#!/usr/bin/env python3

import re
import math
from collections import Counter
from pathlib import Path

import pymysql


BASE_DIR = Path(__file__).resolve().parent.parent
DB_CONFIG = BASE_DIR / "config" / "database.php"

# ============================================================
# SETTINGS
# ============================================================

MAX_ITEMS = 200

# Only compare news reasonably close in time.
MAX_TIME_DISTANCE_HOURS = 12

# Similarity needed to join an existing story.
MATCH_THRESHOLD = 0.34

# Strong overlap can compensate for short texts.
STRONG_TOKEN_THRESHOLD = 0.48


# ============================================================
# PERSIAN NORMALIZATION
# ============================================================

STOPWORDS = {
    "و", "در", "به", "از", "که", "این", "آن", "را", "با",
    "برای", "یک", "است", "بود", "شد", "شده", "می", "های",
    "هم", "تا", "بر", "یا", "اما", "اگر", "ما", "من",
    "او", "آنها", "آن‌ها", "خود", "کرد", "کرده", "کند",
    "گفت", "میگوید", "می‌گوید", "درباره", "پس", "بعد",
    "قبل", "امروز", "دیروز", "همچنین", "دیگر", "دارد",
    "دارند", "خواهد", "خواهند", "نیز", "فقط", "خیلی",
    "بسیار", "شما", "هر", "روی", "همین", "هنوز",
    "شوند", "شود", "کردند", "کنند", "کنیم", "کردیم",
    "نیست", "هست", "هستند"
}


def normalize_text(text):

    text = text or ""

    # Remove Telegram usernames
    text = re.sub(
        r"@[A-Za-z0-9_]+",
        " ",
        text
    )

    # Remove URLs
    text = re.sub(
        r"https?://\S+|www\.\S+",
        " ",
        text
    )

    # Persian/Arabic character normalization
    text = (
        text
        .replace("ي", "ی")
        .replace("ى", "ی")
        .replace("ك", "ک")
        .replace("ۀ", "ه")
        .replace("ة", "ه")
        .replace("\u200c", " ")
    )

    text = text.lower()

    # Keep Persian/English letters + numbers
    text = re.sub(
        r"[^آ-یa-z0-9۰-۹\s]",
        " ",
        text
    )

    text = re.sub(
        r"\s+",
        " ",
        text
    ).strip()

    return text


def tokenize(text):

    normalized = normalize_text(text)

    tokens = []

    for token in normalized.split():

        token = token.strip()

        if len(token) < 2:
            continue

        if token in STOPWORDS:
            continue

        tokens.append(token)

    return tokens


# ============================================================
# SIMILARITY
# ============================================================

def weighted_jaccard(tokens_a, tokens_b):

    if not tokens_a or not tokens_b:
        return 0.0

    a = Counter(tokens_a)
    b = Counter(tokens_b)

    common = set(a) & set(b)
    all_tokens = set(a) | set(b)

    numerator = sum(
        min(a[x], b[x])
        for x in common
    )

    denominator = sum(
        max(a[x], b[x])
        for x in all_tokens
    )

    if denominator == 0:
        return 0.0

    return numerator / denominator


def containment(tokens_a, tokens_b):

    a = set(tokens_a)
    b = set(tokens_b)

    if not a or not b:
        return 0.0

    common = len(a & b)

    return common / min(
        len(a),
        len(b)
    )


def important_overlap(tokens_a, tokens_b):

    a = {
        x for x in tokens_a
        if len(x) >= 4
    }

    b = {
        x for x in tokens_b
        if len(x) >= 4
    }

    if not a or not b:
        return 0.0

    common = a & b

    return len(common) / min(
        len(a),
        len(b)
    )


def similarity(item_a, item_b):

    ta = item_a["tokens"]
    tb = item_b["tokens"]

    jac = weighted_jaccard(
        ta,
        tb
    )

    cont = containment(
        ta,
        tb
    )

    important = important_overlap(
        ta,
        tb
    )

    # Weighted score
    score = (
        jac * 0.40
        + cont * 0.35
        + important * 0.25
    )

    return {
        "score": score,
        "jaccard": jac,
        "containment": cont,
        "important": important
    }


# ============================================================
# DB CONFIG
# ============================================================

def load_db_config():

    text = DB_CONFIG.read_text(
        errors="ignore"
    )

    def find(pattern, default=None):

        match = re.search(
            pattern,
            text,
            re.I
        )

        if not match:
            return default

        return match.group(1)

    host = find(
        r"""['"]host['"]\s*=>\s*['"]([^'"]+)""",
        "localhost"
    )

    database = find(
        r"""['"](?:database|dbname)['"]\s*=>\s*['"]([^'"]+)"""
    )

    user = find(
        r"""['"](?:username|user)['"]\s*=>\s*['"]([^'"]+)"""
    )

    password = find(
        r"""['"](?:password|pass)['"]\s*=>\s*['"]([^'"]*)""",
        ""
    )

    if not database or not user:
        raise RuntimeError(
            "Could not read DB config."
        )

    return {
        "host": host,
        "user": user,
        "password": password,
        "database": database,
        "charset": "utf8mb4",
        "cursorclass": pymysql.cursors.DictCursor
    }


# ============================================================
# LOAD RAW ITEMS
# ============================================================

def load_items(db):

    with db.cursor() as cur:

        cur.execute(
            """
            SELECT
                nfi.id,
                nfi.news_source_id,
                nfi.source_id,
                nfi.source_message_id,
                nfi.text_content,
                nfi.content_type,
                nfi.media_type,
                nfi.published_at,
                s.username
            FROM news_feed_items nfi
            LEFT JOIN sources s
                ON s.id = nfi.source_id
            WHERE nfi.feed_status = 'raw'
            ORDER BY
                nfi.published_at ASC,
                nfi.id ASC
            LIMIT %s
            """,
            (
                MAX_ITEMS,
            )
        )

        rows = cur.fetchall()

    result = []

    for row in rows:

        row["tokens"] = tokenize(
            row.get("text_content")
        )

        result.append(row)

    return result


# ============================================================
# TIME CHECK
# ============================================================

def time_distance_hours(a, b):

    da = a.get("published_at")
    db = b.get("published_at")

    if not da or not db:
        return 0

    return abs(
        (da - db).total_seconds()
    ) / 3600


# ============================================================
# STORY MATCHER
# ============================================================

def can_compare(item, story):

    primary = story["primary"]

    hours = time_distance_hours(
        item,
        primary
    )

    return hours <= MAX_TIME_DISTANCE_HOURS


def match_against_story(item, story):

    # Compare against every item already in the story.
    # This prevents a later update from being rejected just
    # because wording differs from the first message.

    best = None

    for existing in story["items"]:

        result = similarity(
            item,
            existing
        )

        if (
            best is None
            or result["score"] > best["score"]
        ):
            best = result

    return best


def should_match(result):

    if not result:
        return False

    if result["score"] >= MATCH_THRESHOLD:
        return True

    if (
        result["containment"] >= STRONG_TOKEN_THRESHOLD
        and result["important"] >= 0.35
    ):
        return True

    return False


def build_stories(items):

    stories = []

    for item in items:

        best_story = None
        best_result = None

        for story in stories:

            if not can_compare(
                item,
                story
            ):
                continue

            result = match_against_story(
                item,
                story
            )

            if not should_match(result):
                continue

            if (
                best_result is None
                or result["score"] > best_result["score"]
            ):
                best_story = story
                best_result = result

        if best_story is None:

            stories.append({
                "primary": item,
                "items": [item],
                "matches": []
            })

        else:

            best_story["items"].append(
                item
            )

            best_story["matches"].append({
                "item_id": item["id"],
                "score": best_result["score"],
                "jaccard": best_result["jaccard"],
                "containment": best_result["containment"],
                "important": best_result["important"]
            })

    return stories


# ============================================================
# DISPLAY
# ============================================================

def short_text(text, limit=110):

    text = normalize_text(text)

    if len(text) <= limit:
        return text

    return text[:limit] + "..."


def main():

    db = pymysql.connect(
        **load_db_config()
    )

    try:

        items = load_items(db)

        stories = build_stories(
            items
        )

        print()
        print(
            "NEWS STORY MATCHER V1 — DRY RUN"
        )
        print(
            "=============================================="
        )
        print(
            f"Raw Items:         {len(items)}"
        )
        print(
            f"Detected Stories:  {len(stories)}"
        )
        print(
            f"Grouped Items:     "
            f"{sum(len(x['items']) for x in stories if len(x['items']) > 1)}"
        )
        print(
            f"Multi-item Stories:"
            f" {sum(1 for x in stories if len(x['items']) > 1)}"
        )
        print(
            "DB Writes:         0"
        )
        print(
            "=============================================="
        )

        for number, story in enumerate(
            stories,
            start=1
        ):

            primary = story["primary"]

            print()
            print(
                f"STORY #{number}"
            )
            print(
                "----------------------------------------------"
            )
            print(
                f"Items:       {len(story['items'])}"
            )
            print(
                f"Primary:     {primary['id']}"
            )
            print(
                f"First Seen:  {primary['published_at']}"
            )
            print(
                "Headline:    "
                + short_text(
                    primary.get("text_content")
                )
            )

            for item in story["items"]:

                print(
                    f"  - Feed #{item['id']} "
                    f"| msg {item['source_message_id']} "
                    f"| @{item.get('username') or '-'}"
                )

                print(
                    "    "
                    + short_text(
                        item.get("text_content"),
                        150
                    )
                )

            if story["matches"]:

                print(
                    "  Similarities:"
                )

                for match in story["matches"]:

                    print(
                        "    "
                        f"Feed #{match['item_id']} → "
                        f"{match['score']:.3f} "
                        f"(containment "
                        f"{match['containment']:.3f})"
                    )

        print()
        print(
            "=============================================="
        )
        print(
            "DRY RUN COMPLETE — DATABASE UNCHANGED ✅"
        )

    finally:

        db.close()


if __name__ == "__main__":
    main()