#!/usr/bin/env python3
"""Adapt the official CC-CEDICT export into a read-only, indexed input lexicon.

Dictionary data and additions are CC BY-SA 4.0. App code is separate.
"""
import gzip
import argparse
import hashlib
import json
import re
import sqlite3
from pathlib import Path

ROOT = next((p for p in Path(__file__).resolve().parents if (p / "app-config.json").is_file()), Path.cwd())
SOURCE = ROOT / ".local/research/cedict.txt.gz"
OUTPUT = ROOT / "resources/pinyin.sqlite3"
MAP = {c: digit for digit, chars in enumerate(["abc", "def", "ghi", "jkl", "mno", "pqrs", "tuv", "wxyz"], 2) for c in chars}
COMMON = "我你他的了是不在有好这们吗呢很也就都想说可以没和一要来去会到看对知道喜欢给今天明白时候什么怎么谢谢晚安早上朋友爱工作吃饭现在一起回来真的开心抱歉关系再见"
ADDITIONS = [
    ("你好", "ni hao"), ("谢谢", "xie xie"), ("晚安", "wan an"), ("早上好", "zao shang hao"),
    ("没关系", "mei guan xi"), ("对不起", "dui bu qi"), ("我想你", "wo xiang ni"), ("我喜欢你", "wo xi huan ni"),
    ("你好吗", "ni hao ma"), ("吃饭了吗", "chi fan le ma"), ("等一下", "deng yi xia"),
    ("一会见", "yi hui jian"), ("今天怎么样", "jin tian zen me yang"), ("我很好", "wo hen hao"),
    ("好的", "hao de"), ("可以", "ke yi"), ("没有", "mei you"), ("真的", "zhen de"),
    ("开心", "kai xin"), ("加油", "jia you"), ("我也是", "wo ye shi"), ("有时间吗", "you shi jian ma"),
    ("周末见", "zhou mo jian"), ("注意休息", "zhu yi xiu xi"), ("辛苦了", "xin ku le"),
    ("我们", "wo men"), ("你们", "ni men"), ("他们", "ta men"), ("她们", "ta men"),
    ("我的", "wo de"), ("你的", "ni de"), ("今天", "jin tian"), ("明天", "ming tian"),
    ("现在", "xian zai"), ("刚才", "gang cai"), ("已经", "yi jing"), ("还是", "hai shi"),
    ("因为", "yin wei"), ("所以", "suo yi"), ("但是", "dan shi"), ("然后", "ran hou"),
    ("如果", "ru guo"), ("可能", "ke neng"), ("应该", "ying gai"), ("觉得", "jue de"),
    ("知道", "zhi dao"), ("不知道", "bu zhi dao"), ("为什么", "wei shen me"), ("怎么", "zen me"),
    ("什么", "shen me"), ("怎么样", "zen me yang"), ("什么时候", "shen me shi hou"),
    ("没事", "mei shi"), ("没问题", "mei wen ti"), ("没什么", "mei shen me"),
    ("是不是", "shi bu shi"), ("有没有", "you mei you"), ("要不要", "yao bu yao"),
    ("好的呀", "hao de ya"), ("好吧", "hao ba"), ("好像", "hao xiang"), ("好久不见", "hao jiu bu jian"),
    ("喜欢", "xi huan"), ("一起", "yi qi"), ("聊天", "liao tian"), ("事情", "shi qing"),
    ("工作", "gong zuo"), ("吃饭", "chi fan"), ("休息", "xiu xi"), ("睡觉", "shui jiao"),
    ("回家", "hui jia"), ("上班", "shang ban"), ("下班", "xia ban"), ("到家", "dao jia"),
    ("想你", "xiang ni"), ("想想", "xiang xiang"), ("等你", "deng ni"), ("谢谢你", "xie xie ni"),
    ("不客气", "bu ke qi"), ("不用", "bu yong"), ("不用谢", "bu yong xie"), ("不好意思", "bu hao yi si"),
    ("我觉得", "wo jue de"), ("我知道", "wo zhi dao"), ("我不知道", "wo bu zhi dao"),
    ("我想", "wo xiang"), ("我在", "wo zai"), ("我是", "wo shi"), ("我可以", "wo ke yi"),
    ("你在干嘛", "ni zai gan ma"), ("你在哪里", "ni zai na li"), ("你说", "ni shuo"),
    ("真的吗", "zhen de ma"), ("差不多", "cha bu duo"), ("这边", "zhe bian"), ("那边", "na bian"),
    ("这里", "zhe li"), ("那里", "na li"), ("这个", "zhe ge"), ("那个", "na ge"),
    ("这样", "zhe yang"), ("那样", "na yang"), ("最近", "zui jin"), ("有点", "you dian"),
    ("非常", "fei chang"), ("一直", "yi zhi"), ("其实", "qi shi"), ("一定", "yi ding"),
    ("当然", "dang ran"), ("一下", "yi xia"), ("一会儿", "yi hui er"), ("马上", "ma shang"),
    ("开心吗", "kai xin ma"), ("早点休息", "zao dian xiu xi"), ("明天见", "ming tian jian"),
]


def normalize(value):
    return re.sub(r"[^a-zv]", "", value.lower().replace("u:", "v").replace("ü", "v"))


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--source", type=Path, default=SOURCE, help="Official CC-CEDICT gzip export")
    parser.add_argument("--output", type=Path, default=OUTPUT, help="Adapted SQLite database")
    args = parser.parse_args()
    source, output = args.source, args.output
    if not source.exists():
        raise SystemExit("Download the official CC-CEDICT export from https://www.mdbg.net/chinese/dictionary?page=cedict and pass --source cedict.txt.gz --output pinyin.sqlite3. Data is CC BY-SA 4.0.")
    output.parent.mkdir(parents=True, exist_ok=True)
    temporary = output.with_suffix(".building.sqlite3")
    temporary.unlink(missing_ok=True)
    db = sqlite3.connect(temporary)
    db.executescript("CREATE TABLE words(word TEXT NOT NULL,pinyin TEXT NOT NULL,letters TEXT NOT NULL,digits TEXT NOT NULL,rank INTEGER NOT NULL,PRIMARY KEY(word,letters));")
    counts = 0
    for line in gzip.decompress(source.read_bytes()).decode("utf-8").splitlines():
        match = re.match(r"^\S+ (\S+) \[([^]]+)\]", line)
        if not match: continue
        word, pinyin = match.groups()
        letters = normalize(pinyin)
        if not 1 <= len(word) <= 12 or not letters or not all("\u3400" <= c <= "\u9fff" for c in word): continue
        rank = 1000
        if len(word) == 1 and word in COMMON: rank = 100000 - COMMON.index(word) * 100
        digits = "".join(str(MAP[c]) for c in letters)
        db.execute("INSERT OR IGNORE INTO words VALUES(?,?,?,?,?)", (word, re.sub(r"[1-5]", "", pinyin).replace("u:", "ü"), letters, digits, rank))
        counts += 1
    for index, (word, pinyin) in enumerate(ADDITIONS):
        letters = normalize(pinyin)
        db.execute("INSERT OR REPLACE INTO words VALUES(?,?,?,?,?)", (word, pinyin, letters, "".join(str(MAP[c]) for c in letters), 90000 - index * 100))
    db.executescript("CREATE INDEX words_letters ON words(letters); CREATE INDEX words_digits ON words(digits); PRAGMA user_version=1;")
    db.commit()
    rows = db.execute("SELECT count(*) FROM words").fetchone()[0]
    assert rows > 90000
    assert db.execute("SELECT word FROM words WHERE digits='64426' AND word='你好'").fetchone()
    db.close()
    temporary.replace(output)
    report = {"source": "CC-CEDICT", "sourceURL": "https://www.mdbg.net/chinese/dictionary?page=cedict", "license": "CC-BY-SA-4.0",
              "sourceSHA256": hashlib.sha256(source.read_bytes()).hexdigest(), "entries": rows, "bytes": output.stat().st_size,
              "adaptations": "Simplified Hanzi, normalized readings, T9 index, common-character weights and original conversational additions."}
    (output.parent / "pinyin-manifest.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
    print(json.dumps(report, ensure_ascii=False))


if __name__ == "__main__": main()
