#!/usr/bin/env python3

import re
import math
from collections import Counter
from pathlib import Path

import pymysql


BASE_DIR = Path(__file__).resolve().parent.parent
DB_CONFIG = BASE_DIR / "config" / "database.php"

# ============================================================
# SETTINGS
# ============================================================

MAX_ITEMS = 500

# Only compare news reasonably close in time.
MAX_TIME_DISTANCE_HOURS = 12

# Similarity needed to join an existing story.
MATCH_THRESHOLD = 0.34

# Strong overlap can compensate for short texts.
STRONG_TOKEN_THRESHOLD = 0.48


# ============================================================
# PERSIAN NORMALIZATION
# ============================================================

STOPWORDS = {
    "و", "در", "به", "از", "که", "این", "آن", "را", "با",
    "برای", "یک", "است", "بود", "شد", "شده", "می", "های",
    "هم", "تا", "بر", "یا", "اما", "اگر", "ما", "من",
    "او", "آنها", "آن‌ها", "خود", "کرد", "کرده", "کند",
    "گفت", "میگوید", "می‌گوید", "درباره", "پس", "بعد",
    "قبل", "امروز", "دیروز", "همچنین", "دیگر", "دارد",
    "دارند", "خواهد", "خواهند", "نیز", "فقط", "خیلی",
    "بسیار", "شما", "هر", "روی", "همین", "هنوز",
    "شوند", "شود", "کردند", "کنند", "کنیم", "کردیم",
    "نیست", "هست", "هستند"
}


def normalize_text(text):

    text = text or ""

    # Remove Telegram usernames
    text = re.sub(
        r"@[A-Za-z0-9_]+",
        " ",
        text
    )

    # Remove URLs
    text = re.sub(
        r"https?://\S+|www\.\S+",
        " ",
        text
    )

    # Persian/Arabic character normalization
    text = (
        text
        .replace("ي", "ی")
        .replace("ى", "ی")
        .replace("ك", "ک")
        .replace("ۀ", "ه")
        .replace("ة", "ه")
        .replace("\u200c", " ")
    )

    text = text.lower()

    # Keep Persian/English letters + numbers
    text = re.sub(
        r"[^آ-یa-z0-9۰-۹\s]",
        " ",
        text
    )

    text = re.sub(
        r"\s+",
        " ",
        text
    ).strip()

    return text


def tokenize(text):

    normalized = normalize_text(text)

    tokens = []

    for token in normalized.split():

        token = token.strip()

        if len(token) < 2:
            continue

        if token in STOPWORDS:
            continue

        tokens.append(token)

    return tokens


# ============================================================
# SIMILARITY
# ============================================================

def weighted_jaccard(tokens_a, tokens_b):

    if not tokens_a or not tokens_b:
        return 0.0

    a = Counter(tokens_a)
    b = Counter(tokens_b)

    common = set(a) & set(b)
    all_tokens = set(a) | set(b)

    numerator = sum(
        min(a[x], b[x])
        for x in common
    )

    denominator = sum(
        max(a[x], b[x])
        for x in all_tokens
    )

    if denominator == 0:
        return 0.0

    return numerator / denominator


def containment(tokens_a, tokens_b):

    a = set(tokens_a)
    b = set(tokens_b)

    if not a or not b:
        return 0.0

    common = len(a & b)

    return common / min(
        len(a),
        len(b)
    )


def important_overlap(tokens_a, tokens_b):

    a = {
        x for x in tokens_a
        if len(x) >= 4
    }

    b = {
        x for x in tokens_b
        if len(x) >= 4
    }

    if not a or not b:
        return 0.0

    common = a & b

    return len(common) / min(
        len(a),
        len(b)
    )


def similarity(item_a, item_b):

    ta = item_a["tokens"]
    tb = item_b["tokens"]

    jac = weighted_jaccard(
        ta,
        tb
    )

    cont = containment(
        ta,
        tb
    )

    important = important_overlap(
        ta,
        tb
    )

    # Weighted score
    score = (
        jac * 0.40
        + cont * 0.35
        + important * 0.25
    )

    return {
        "score": score,
        "jaccard": jac,
        "containment": cont,
        "important": important
    }


# ============================================================
# DB CONFIG
# ============================================================

def load_db_config():

    content = DB_CONFIG.read_text(
        encoding="utf-8",
        errors="ignore"
    )

    patterns = {
        "host":
            r"\$dbHost\s*=\s*['\"]([^'\"]+)['\"]",

        "database":
            r"\$dbName\s*=\s*['\"]([^'\"]+)['\"]",

        "user":
            r"\$dbUser\s*=\s*['\"]([^'\"]+)['\"]",

        "password":
            r"\$dbPass\s*=\s*['\"]([^'\"]*)['\"]",
    }

    result = {}

    for key, pattern in patterns.items():

        match = re.search(
            pattern,
            content
        )

        if not match:
            raise RuntimeError(
                f"Could not read DB config: {key}"
            )

        result[key] = match.group(1)

    return {
        "host": result["host"],
        "user": result["user"],
        "password": result["password"],
        "database": result["database"],
        "charset": "utf8mb4",
        "cursorclass": pymysql.cursors.DictCursor
    }


# ============================================================
# LOAD RAW ITEMS
# ============================================================

def load_items(db):

    with db.cursor() as cur:

        cur.execute(
            """
            SELECT
                nfi.id,
                nfi.news_source_id,
                nfi.source_id,
                nfi.source_message_id,
                nfi.text_content,
                nfi.content_type,
                nfi.media_type,
                nfi.published_at,
                s.username
            FROM news_feed_items nfi
            LEFT JOIN sources s
                ON s.id = nfi.source_id
            WHERE nfi.feed_status = 'raw'
            ORDER BY
                nfi.published_at ASC,
                nfi.id ASC
            LIMIT %s
            """,
            (
                MAX_ITEMS,
            )
        )

        rows = cur.fetchall()

    result = []

    for row in rows:

        row["tokens"] = tokenize(
            row.get("text_content")
        )

        result.append(row)

    return result


# ============================================================
# TIME CHECK
# ============================================================

def time_distance_hours(a, b):

    da = a.get("published_at")
    db = b.get("published_at")

    if not da or not db:
        return 0

    return abs(
        (da - db).total_seconds()
    ) / 3600


# ============================================================
# STORY MATCHER
# ============================================================

def can_compare(item, story):

    primary = story["primary"]

    hours = time_distance_hours(
        item,
        primary
    )

    return hours <= MAX_TIME_DISTANCE_HOURS


def exactish_duplicate(result):
    """
    Very strong textual match.
    Intended for the same news copied/reposted with only
    minor formatting/source differences.
    """

    if not result:
        return False

    if result["score"] >= 0.72:
        return True

    if (
        result["containment"] >= 0.88
        and result["important"] >= 0.70
    ):
        return True

    return False


def normal_story_match(result):
    """
    Normal semantic/textual match.

    Stricter than V1 because generic overlap such as:
      Trump + Iran
      oil + price
      Macron + France
      Strait of Hormuz
    must not be enough by itself.
    """

    if not result:
        return False

    if (
        result["score"] >= 0.43
        and result["containment"] >= 0.52
        and result["important"] >= 0.42
    ):
        return True

    if (
        result["containment"] >= 0.68
        and result["important"] >= 0.52
        and result["jaccard"] >= 0.24
    ):
        return True

    return False


def story_similarity(item, story):
    """
    V2 anti-chain matcher.

    A new item is evaluated against:
      1) the primary story item
      2) every existing member

    But matching one random member is no longer automatically
    enough to join a large story.
    """

    primary_result = similarity(
        item,
        story["primary"]
    )

    member_results = []

    for existing in story["items"]:

        result = similarity(
            item,
            existing
        )

        member_results.append({
            "item_id": existing["id"],
            **result
        })

    member_results.sort(
        key=lambda x: x["score"],
        reverse=True
    )

    best = (
        member_results[0]
        if member_results
        else None
    )

    second = (
        member_results[1]
        if len(member_results) >= 2
        else None
    )

    return {
        "primary": primary_result,
        "best": best,
        "second": second,
        "members": member_results
    }


def should_match_story(item, story, result):
    """
    Decide whether ITEM really belongs to STORY.

    Rules:

    A) Exact/near duplicate of primary -> accept.

    B) Strong normal match with primary -> accept.

    C) For a one-item story, strong match with that item -> accept.

    D) For an existing multi-item story:
       matching only one member is NOT sufficient unless the match
       is extremely strong. Otherwise the candidate needs support
       from more than one member.

    This prevents chain-merging.
    """

    if not result:
        return False, "no-result"

    primary = result["primary"]
    best = result["best"]
    second = result["second"]

    # --------------------------------------------------------
    # A) Near duplicate of primary
    # --------------------------------------------------------

    if exactish_duplicate(primary):
        return True, "primary-exact"

    # --------------------------------------------------------
    # B) Proper match against primary
    # --------------------------------------------------------

    if normal_story_match(primary):
        return True, "primary-match"

    # --------------------------------------------------------
    # C) Story currently has only one item
    # --------------------------------------------------------

    if len(story["items"]) == 1:

        if best and normal_story_match(best):
            return True, "single-story-match"

        return False, "single-story-reject"

    # --------------------------------------------------------
    # D1) Extremely strong duplicate of an existing member.
    #
    # Useful when later channels copy an update almost verbatim
    # even if that update differs substantially from the primary.
    # --------------------------------------------------------

    if best and exactish_duplicate(best):

        # Do not allow a merely topical member to bridge stories.
        # The threshold here is deliberately very high.
        if (
            best["score"] >= 0.82
            or (
                best["containment"] >= 0.95
                and best["important"] >= 0.82
            )
        ):
            return True, "member-exact"

    # --------------------------------------------------------
    # D2) Consensus support.
    #
    # Candidate should resemble at least two existing members.
    # This prevents:
    #
    # A ~ B
    # B ~ C
    # therefore incorrectly A ~ C
    # --------------------------------------------------------

    if best and second:

        best_ok = normal_story_match(best)

        second_support = (
            second["score"] >= 0.36
            and second["containment"] >= 0.46
            and second["important"] >= 0.36
        )

        if best_ok and second_support:
            return True, "member-consensus"

    return False, "anti-chain-reject"


def build_stories(items):

    stories = []

    for item in items:

        best_story = None
        best_result = None
        best_reason = None
        best_score = -1.0

        for story in stories:

            if not can_compare(
                item,
                story
            ):
                continue

            result = story_similarity(
                item,
                story
            )

            accepted, reason = should_match_story(
                item,
                story,
                result
            )

            if not accepted:
                continue

            primary_score = (
                result["primary"]["score"]
                if result.get("primary")
                else 0.0
            )

            member_score = (
                result["best"]["score"]
                if result.get("best")
                else 0.0
            )

            # Prefer stories whose PRIMARY also agrees.
            # This makes accidental member bridges less attractive.
            candidate_score = (
                primary_score * 0.65
                + member_score * 0.35
            )

            if candidate_score > best_score:

                best_score = candidate_score
                best_story = story
                best_result = result
                best_reason = reason

        if best_story is None:

            stories.append({
                "primary": item,
                "items": [item],
                "matches": []
            })

        else:

            best_story["items"].append(
                item
            )

            best = best_result.get("best") or {}
            primary = best_result.get("primary") or {}

            best_story["matches"].append({
                "item_id": item["id"],
                "score": best.get(
                    "score",
                    primary.get("score", 0.0)
                ),
                "jaccard": best.get(
                    "jaccard",
                    primary.get("jaccard", 0.0)
                ),
                "containment": best.get(
                    "containment",
                    primary.get("containment", 0.0)
                ),
                "important": best.get(
                    "important",
                    primary.get("important", 0.0)
                ),
                "primary_score": primary.get(
                    "score",
                    0.0
                ),
                "matched_to": best.get(
                    "item_id"
                ),
                "reason": best_reason
            })

    return stories


# ============================================================
# DISPLAY
# ============================================================

def short_text(text, limit=110):

    text = normalize_text(text)

    if len(text) <= limit:
        return text

    return text[:limit] + "..."


def main():

    db = pymysql.connect(
        **load_db_config()
    )

    try:

        items = load_items(db)

        stories = build_stories(
            items
        )

        print()
        print(
            "NEWS STORY MATCHER V2 — ANTI-CHAIN DRY RUN"
        )
        print(
            "=============================================="
        )
        print(
            f"Raw Items:         {len(items)}"
        )
        print(
            f"Detected Stories:  {len(stories)}"
        )
        print(
            f"Grouped Items:     "
            f"{sum(len(x['items']) for x in stories if len(x['items']) > 1)}"
        )
        print(
            f"Multi-item Stories:"
            f" {sum(1 for x in stories if len(x['items']) > 1)}"
        )
        print(
            "DB Writes:         0"
        )
        print(
            "=============================================="
        )

        for number, story in enumerate(
            stories,
            start=1
        ):

            primary = story["primary"]

            print()
            print(
                f"STORY #{number}"
            )
            print(
                "----------------------------------------------"
            )
            print(
                f"Items:       {len(story['items'])}"
            )
            print(
                f"Primary:     {primary['id']}"
            )
            print(
                f"First Seen:  {primary['published_at']}"
            )
            print(
                "Headline:    "
                + short_text(
                    primary.get("text_content")
                )
            )

            for item in story["items"]:

                print(
                    f"  - Feed #{item['id']} "
                    f"| msg {item['source_message_id']} "
                    f"| @{item.get('username') or '-'}"
                )

                print(
                    "    "
                    + short_text(
                        item.get("text_content"),
                        150
                    )
                )

            if story["matches"]:

                print(
                    "  Similarities:"
                )

                for match in story["matches"]:

                    print(
                        "    "
                        f"Feed #{match['item_id']} → "
                        f"{match['score']:.3f} "
                        f"(containment "
                        f"{match['containment']:.3f})"
                    )

        print()
        print(
            "=============================================="
        )
        print(
            "DRY RUN COMPLETE — DATABASE UNCHANGED ✅"
        )

    finally:

        db.close()


if __name__ == "__main__":
    main()