import re
import os
from bs4 import BeautifulSoup
from tqdm import tqdm  # 导入进度条库

# 配置路径
INPUT_FILE = r"C:\Users\Administrator\Desktop\data\英汉大词典_output.txt"
OUTPUT_FILE = r"C:\Users\Administrator\Desktop\data\英汉大词典_processed.txt"

def clean_headword(hw):
    """规范化词头：去除音节分隔符和首尾空格"""
    return hw.replace('·', '').strip()

def process_dictionary():
    if not os.path.exists(INPUT_FILE):
        print(f"错误: 找不到输入文件 {INPUT_FILE}")
        return

    print("正在读取文件并进行初步分割...")
    with open(INPUT_FILE, 'r', encoding='utf-8') as f:
        content = f.read()

    # 以 </>\n 分割词条
    raw_entries = [e for e in content.split('</>\n') if e.strip()]
    total_count = len(raw_entries)
    
    processed_entries = []
    redirects = {} 
    existing_hws = set()

    # 使用 tqdm 包装循环以实现可视化进度
    print(f"开始处理 {total_count} 个词条...")
    for entry in tqdm(raw_entries, desc="数据清洗中", unit="词条"):
        lines = entry.strip().split('\n', 1)
        if len(lines) < 2:
            continue
            
        raw_hw = lines[0].strip()
        html_content = lines[1]
        
        # 6. 规范化词头
        clean_hw = clean_headword(raw_hw)
        existing_hws.add(clean_hw)
        
        # 使用 BeautifulSoup 解析 (lxml 解析速度更快，如果没有安装则默认 html.parser)
        try:
            soup = BeautifulSoup(html_content, 'lxml')
        except:
            soup = BeautifulSoup(html_content, 'html.parser')
        
        # 1 & 4. 去除多媒体、图标和外部资源依赖
        for tag in soup.find_all(['img', 'link', 'style']):
            tag.decompose()
        
        for pr in soup.find_all('pr'):
            if pr.has_attr('soundfile'):
                del pr['soundfile']
        
        # 2 & 5. 去除冗余 div、特定参数及样式解耦
        redundant_selectors = [
            'params', '.paramEId', '.paramEHw', 
            '#blankDiv', '#flexGapDiv', '.toDetailBtn'
        ]
        for selector in redundant_selectors:
            for tag in soup.select(selector):
                tag.decompose()

        # 9. 提取屈折变化和派生词用于生成重定向
        # 扫描屈折变化 (nlp/pl)
        for nlp in soup.find_all(['pl', 'nlp']):
            variant = clean_headword(nlp.get_text())
            if variant and variant != clean_hw:
                redirects[variant] = clean_hw
        
        # 扫描派生词 (derivative)
        for der in soup.select('.derivative .hwSpan, .derivative .l'):
            der_word = clean_headword(der.get_text())
            if der_word and der_word != clean_hw:
                redirects[der_word] = clean_hw

        # 3. 去除过深嵌套层级 (解包无意义的容器)
        for div in soup.find_all('div'):
            # 如果 div 没有 class，或者是纯容器，则将其内容移出并删除该 div 标签本身
            if not div.has_attr('class') or not div.get('class'):
                div.unwrap()

        # 8. 规范化 HTML 输出并去除异常行终止符
        cleaned_html = soup.decode(formatter="minimal").replace('\r', '').strip()
        
        # 拼装词条 (7. 隐式通过 strip 和 join 处理了空行)
        processed_entries.append(f"{clean_hw}\n{cleaned_html}\n</>")

    # 处理重定向词条
    print("正在生成独立重定向条目...")
    link_entries = []
    # 过滤掉已经是主词头的变体，防止循环重定向
    for variant, target in tqdm(redirects.items(), desc="链接提取中", unit="链接"):
        if variant not in existing_hws:
            link_entries.append(f"{variant}\n@@@LINK={target}\n</>")
            existing_hws.add(variant)

    final_data = processed_entries + link_entries

    # 写入文件
    print(f"正在保存处理后的数据到: {OUTPUT_FILE}")
    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        # 使用进度条写入文件，防止大文件写入时程序看似“卡死”
        for entry in tqdm(final_data, desc="写入磁盘", unit="词条"):
            f.write(entry + "\n")

    print("\n✅ 所有处理已完成！")

if __name__ == "__main__":
    process_dictionary()