#!/usr/bin/env python3
"""Build a bounded, indexed English model from licensed, pinned SymSpell data."""
import argparse
import hashlib
import json
import re
import sqlite3
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
REVISION = 'c239062ae02961df18ab7da1671d01b4388204e0'
FILES = {
 'frequency_dictionary_en_82_765.txt': 'c604e1121e398ae7c7fbf777f11e0a0f2fa66eda932cb9fba1321466cf3acd7b',
 'frequency_bigramdictionary_en_243_342.txt': 'fd892a160184101dd7ae807ac5a302d01fcea1c47304181a8ed7ed9c94545bcd',
}

def main():
 p=argparse.ArgumentParser(description=__doc__)
 p.add_argument('--source-dir',type=Path,default=ROOT/'.local/research/english')
 p.add_argument('--output',type=Path,default=ROOT/'resources/english.sqlite3')
 a=p.parse_args()
 for name,digest in FILES.items():
  path=a.source_dir/name
  if not path.exists() or hashlib.sha256(path.read_bytes()).hexdigest()!=digest:
   raise SystemExit(f'Provide the pinned source {name} from https://github.com/wolfgarbe/SymSpell/tree/{REVISION}/SymSpell; checksum mismatch or missing file.')
 words={}
 for line in (a.source_dir/'frequency_dictionary_en_82_765.txt').read_text(encoding='utf-8-sig').splitlines():
  word,count=line.rsplit(maxsplit=1)
  if len(word)<=40 and re.fullmatch("[a-z]+(?:'[a-z]+)*",word):words[word]=max(words.get(word,0),int(count))
 pairs={}
 for line in (a.source_dir/'frequency_bigramdictionary_en_243_342.txt').read_text(encoding='utf-8-sig').splitlines():
  first,second,count=line.split()
  if first in words and second in words:pairs[first,second]=max(pairs.get((first,second),0),int(count))
 ranked={}
 for (first,second),count in pairs.items():ranked.setdefault(first,[]).append((second,count))
 rows=[]
 for first,values in sorted(ranked.items()):
  rows.extend((first,word,count) for word,count in sorted(values,key=lambda x:(-x[1],x[0]))[:16])
 a.output.parent.mkdir(parents=True,exist_ok=True)
 temporary=a.output.with_suffix('.building.sqlite3');temporary.unlink(missing_ok=True)
 db=sqlite3.connect(temporary)
 db.executescript('CREATE TABLE words(word TEXT PRIMARY KEY,frequency INTEGER NOT NULL) WITHOUT ROWID; CREATE TABLE pairs(previous TEXT NOT NULL,next TEXT NOT NULL,frequency INTEGER NOT NULL,PRIMARY KEY(previous,next)) WITHOUT ROWID;')
 db.executemany('INSERT INTO words VALUES(?,?)',sorted(words.items()))
 db.executemany('INSERT INTO pairs VALUES(?,?,?)',rows)
 db.executescript('CREATE INDEX pairs_frequency ON pairs(previous,frequency DESC,next); PRAGMA user_version=1;')
 db.commit();db.close();temporary.replace(a.output)
 manifest={'source':'SymSpell frequency dictionaries','revision':REVISION,'sources':FILES,'words':len(words),'wordPairs':len(rows),'contexts':len(ranked),'bytes':a.output.stat().st_size,'runtime':'Read-only SQLite; no network or user typing history','changes':'ASCII English words/apostrophes, max 40 characters, retained source frequencies, 16 highest-frequency continuations per known word; deterministic sorting','licenses':['SymSpell MIT','Google Books Ngram CC BY 3.0','SCOWL source notices in ENGLISH-DICTIONARY-LICENSE.md']}
 (a.output.parent/'english-manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
 print(json.dumps(manifest))
if __name__=='__main__':main()
