"""用预先分词的中文教学语料实现一种 BM25 形式；不把空格切分当中文分词器。"""
from collections import Counter
import math

# 分词结果由作者给定；真实应用需要一致的中文分析器与编号规范化。
docs = {
    "p204": ["P-204", "质量", "退货", "运费", "商家"],
    "p205": ["P-205", "质量", "退货", "运费", "商家"],
    "invoice": ["发票", "申请", "电子", "发票"],
}
query = ["P-204", "运费"]
N = len(docs)
avg_len = sum(map(len, docs.values())) / N
k1, b = 1.2, 0.75
scores = {}
for doc_id, tokens in docs.items():
    frequencies = Counter(tokens)
    score = 0.0
    for term in dict.fromkeys(query):
        df = sum(term in words for words in docs.values())
        # 使用带 +1 的非负 IDF 形式；其他库的 BM25 变体可能不同。
        idf = math.log(1 + (N - df + 0.5) / (df + 0.5))
        tf = frequencies[term]
        norm = tf + k1 * (1 - b + b * len(tokens) / avg_len)
        score += idf * tf * (k1 + 1) / norm
    scores[doc_id] = score
print(sorted(scores.items(), key=lambda pair: (-pair[1], pair[0])))