import json
import os
import re
import sys

sys.stdout.reconfigure(encoding='utf-8')

# --- Helper function to extract bracket components ---

def extract_balanced_bracket(s, start_idx):
    start_char = s[start_idx]
    if start_char == '【':
        end_chars = ['】', ']']
    elif start_char == '[':
        end_chars = [']', '】']
    elif start_char == '「':
        end_chars = ['」', '】', ']']
    else:
        return None, start_idx
        
    depth = 1
    j = start_idx + 1
    while j < len(s) and depth > 0:
        if s[j] == start_char:
            depth += 1
        elif s[j] in end_chars:
            depth -= 1
        j += 1
    return s[start_idx:j], j

# --- Hierarchical Sense Parsing and Validation Helpers ---

L1_SEQUENCE = ['❶', '❷', '❸', '❹', '❺', '❻', '❼', '❽', '❾', '❿']
L2_SEQUENCE = ['ⓐ', 'ⓑ', 'ⓒ', 'ⓓ', 'ⓔ', 'ⓕ', 'ⓖ', 'ⓗ', 'ⓘ', 'ⓙ']

def get_reference_ranges(text):
    s = list(text)
    brackets = []
    ranges = []
    for i, c in enumerate(s):
        if c in ['(', '（']:
            brackets.append((i, c))
        elif c in [')', '）']:
            if brackets:
                start, start_char = brackets.pop()
                if (start_char == '(' and c == ')') or (start_char == '（' and c == '）'):
                    sub = "".join(s[start : i + 1])
                    if '→' in sub or '☞' in sub or 'に同じ' in sub:
                        ranges.append((start, i + 1))
                        
    # Fallback for unclosed parentheses starting with reference indicators
    for start, start_char in brackets:
        sub = "".join(s[start:])
        if '→' in sub or '☞' in sub or 'に同じ' in sub:
            ranges.append((start, len(s)))
            
    return ranges

def is_inside_ranges(idx, ranges):
    for start, end in ranges:
        if start <= idx < end:
            return True
    return False

def is_reference_marker(idx, text):
    following_text = text[idx + 1:]
    pattern = re.compile(r'^\s*[❶❷❸❹❺❻❼❽❾❿ⓐⓑⓒⓓⓔⓕⓖⓗⓘⓙ\s]*[にと]同じ')
    return bool(pattern.match(following_text))


def validate_sequence_level1(headword, markers, sequence_errors):
    if not markers:
        return
        
    # Check duplicates
    seen = set()
    duplicates = []
    for m in markers:
        if m in seen:
            duplicates.append(m)
        seen.add(m)
    if duplicates:
        sequence_errors.append({
            'headword': headword,
            'error_type': 'duplicate_level1',
            'message': f"Duplicate Level 1 markers: {', '.join(duplicates)}",
            'markers': list(markers)
        })
        
    # Check order
    unique_markers = []
    for m in markers:
        if m not in unique_markers:
            unique_markers.append(m)
            
    expected = L1_SEQUENCE[:len(unique_markers)]
    if unique_markers != expected:
        sequence_errors.append({
            'headword': headword,
            'error_type': 'sequence_mismatch_level1',
            'message': f"Level 1 sequence mismatch. Expected {', '.join(expected)}, got {', '.join(unique_markers)}",
            'markers': list(markers)
        })

def validate_sequence_level2(headword, sense_num, markers, sequence_errors):
    if not markers:
        return
        
    sense_str = f" under Sense {sense_num}" if sense_num else " at top level"
    
    # Check duplicates
    seen = set()
    duplicates = []
    for m in markers:
        if m in seen:
            duplicates.append(m)
        seen.add(m)
    if duplicates:
        sequence_errors.append({
            'headword': headword,
            'error_type': 'duplicate_level2',
            'message': f"Duplicate Level 2 markers{sense_str}: {', '.join(duplicates)}",
            'markers': list(markers)
        })
        
    # Check order
    unique_markers = []
    for m in markers:
        if m not in unique_markers:
            unique_markers.append(m)
            
    expected = L2_SEQUENCE[:len(unique_markers)]
    if unique_markers != expected:
        sequence_errors.append({
            'headword': headword,
            'error_type': 'sequence_mismatch_level2',
            'message': f"Level 2 sequence mismatch{sense_str}. Expected {', '.join(expected)}, got {', '.join(unique_markers)}",
            'markers': list(markers)
        })

def parse_hierarchy_explanation(text, sequence_errors, headword):
    pattern = re.compile(r'([❶❷❸❹❺❻❼❽❾❿ⓐⓑⓒⓓⓔⓕⓖⓗⓘⓙ])')
    ref_ranges = get_reference_ranges(text)
    
    all_matches = list(pattern.finditer(text))
    matches = [m for m in all_matches if not is_inside_ranges(m.start(), ref_ranges) and not is_reference_marker(m.start(), text)]
    
    if not matches:
        return [{'num': '', 'text': text.strip(), 'subs': []}]
        
    senses = []
    first_start = matches[0].start()
    pre_text = text[:first_start].strip()
    if pre_text:
        senses.append({'num': '', 'text': pre_text, 'subs': []})
        
    current_sense = None
    level1_markers = []
    level2_markers_by_sense = {}
    
    def get_level(char):
        if char in '❶❷❸❹❺❻❼❽❾❿':
            return 1
        return 2
        
    for idx, match in enumerate(matches):
        char = match.group(1)
        start = match.start()
        end = match.end()
        
        next_start = matches[idx + 1].start() if idx + 1 < len(matches) else len(text)
        content = text[end:next_start].strip()
        
        level = get_level(char)
        if level == 1:
            current_sense = {'num': char, 'text': content, 'subs': []}
            senses.append(current_sense)
            level1_markers.append(char)
            level2_markers_by_sense[char] = []
        else:
            if not current_sense:
                current_sense = {'num': '', 'text': '', 'subs': []}
                senses.append(current_sense)
                level2_markers_by_sense[''] = []
            
            sub_item = {'num': char, 'text': content}
            current_sense['subs'].append(sub_item)
            
            sense_num_key = current_sense['num']
            if sense_num_key not in level2_markers_by_sense:
                level2_markers_by_sense[sense_num_key] = []
            level2_markers_by_sense[sense_num_key].append(char)
            
    validate_sequence_level1(headword, level1_markers, sequence_errors)
    for sense_num, m_list in level2_markers_by_sense.items():
        validate_sequence_level2(headword, sense_num, m_list, sequence_errors)
        
    return senses

def parse_hierarchy_flat(text, sequence_errors, headword, section_type):
    pattern = re.compile(r'([❶❷❸❹❺❻❼❽❾❿])')
    ref_ranges = get_reference_ranges(text)
    
    all_matches = list(pattern.finditer(text))
    matches = [m for m in all_matches if not is_inside_ranges(m.start(), ref_ranges) and not is_reference_marker(m.start(), text)]
    
    if not matches:
        return [{'num': '', 'text': text.strip()}]
        
    senses = []
    first_start = matches[0].start()
    pre_text = text[:first_start].strip()
    if pre_text:
        senses.append({'num': '', 'text': pre_text})
        
    level1_markers = []
    
    for idx, match in enumerate(matches):
        char = match.group(1)
        start = match.start()
        end = match.end()
        
        next_start = matches[idx + 1].start() if idx + 1 < len(matches) else len(text)
        content = text[end:next_start].strip()
        
        senses.append({'num': char, 'text': content})
        level1_markers.append(char)
        
    if section_type not in ['adverb', 'other']:
        validate_sequence_level1(headword + f" ({section_type})", level1_markers, sequence_errors)
    return senses

def parse_hierarchy_examples(text, sequence_errors, headword, section_type):
    content = text
    m_pref = re.match(r'^(【?例文】?|【?例\s+文】?|例文)\s*', text)
    if m_pref:
        content = text[m_pref.end():]
        
    pattern = re.compile(r'([❶❷❸❹❺❻❼❽❾❿])')
    ref_ranges = get_reference_ranges(content)
    
    all_matches = list(pattern.finditer(content))
    matches = [m for m in all_matches if not is_inside_ranges(m.start(), ref_ranges) and not is_reference_marker(m.start(), content)]
    
    if not matches:
        items = [s.strip() for s in content.split('/') if s.strip()]
        return [{'num': '', 'items': items}]
        
    senses = []
    first_start = matches[0].start()
    pre_text = content[:first_start].strip()
    if pre_text:
        items = [s.strip() for s in pre_text.split('/') if s.strip()]
        if items:
            senses.append({'num': '', 'items': items})
            
    level1_markers = []
    
    for idx, match in enumerate(matches):
        char = match.group(1)
        start = match.start()
        end = match.end()
        
        next_start = matches[idx + 1].start() if idx + 1 < len(matches) else len(content)
        part_text = content[end:next_start].strip()
        
        items = [s.strip() for s in part_text.split('/') if s.strip()]
        senses.append({'num': char, 'items': items})
        level1_markers.append(char)
        
    validate_sequence_level1(headword + f" ({section_type})", level1_markers, sequence_errors)
    return senses

def parse_header_components(header):
    s = header.strip()
    is_redirect = '☞' in s
    
    if is_redirect:
        # Standard format: **kana【kanji】** ☞ Target
        # E.g. **けりかえす【蹴り返す】** ☞ けかえす【蹴返す】
        m = re.match(r'^\*+([^\u300c\u300d\u3010\u3011\[\]]+)【(.*?)】\*+\s*☞\s*(.*)', s)
        if m:
            kana_block = m.group(1).strip()
            kanji_block = m.group(2).strip()
            target = m.group(3).strip()
            kana_forms = [k.strip() for k in re.split(r'[・、/／]', kana_block) if k.strip()]
            kanji_forms = [k.strip() for k in re.split(r'[・、/／]', kanji_block) if k.strip()]
            return {
                'kana': kana_forms,
                'kanji': kanji_forms,
                'v1_v2': None,
                'is_redirect': True,
                'redirect_target': target,
                'explanation_start': ''
            }
        return {
            'kana': [],
            'kanji': [],
            'v1_v2': None,
            'is_redirect': True,
            'redirect_target': s.split('☞')[-1].strip(),
            'explanation_start': ''
        }
        
    # Regular header format: **kana【kanji】**[V1＋V2] rest
    idx_kanji_start = s.find('【')
    if idx_kanji_start == -1:
        return None
    
    kana_block = s[2:idx_kanji_start].strip()
    
    kanji_block, idx_kanji_end = extract_balanced_bracket(s, idx_kanji_start)
    if not kanji_block:
        return None
    
    kanji_content = kanji_block[1:-1].strip()
    
    idx_v1_v2_start = s.find('[', idx_kanji_end)
    if idx_v1_v2_start == -1:
        return None
        
    v1_v2_block, idx_v1_v2_end = extract_balanced_bracket(s, idx_v1_v2_start)
    if not v1_v2_block:
        return None
        
    v1_v2_content = v1_v2_block[1:-1].strip()
    rest = s[idx_v1_v2_end:].strip()
    
    kana_forms = [k.strip() for k in re.split(r'[・、/／]', kana_block) if k.strip()]
    kanji_forms = [k.strip() for k in re.split(r'[・、/／]', kanji_content) if k.strip()]
    
    return {
        'kana': kana_forms,
        'kanji': kanji_forms,
        'v1_v2': v1_v2_content,
        'is_redirect': False,
        'redirect_target': None,
        'explanation_start': rest
    }

def main():
    input_file = r"d:\OCR Project\复合动词\研究社日本語複合動詞活用辞典_gemini_processed.txt"
    output_file = r"d:\OCR Project\复合动词\dictionary.json"
    
    if not os.path.exists(input_file):
        print(f"Error: Input file {input_file} does not exist.")
        return
        
    with open(input_file, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    entries = []
    current_entry = None
    
    # We will use this to track section type
    current_section_type = 'explanation'
    
    for idx, line in enumerate(lines):
        line_strip = line.strip()
        
        # Check if line is header
        # Redirection headers: **kana【kanji】** ☞ ...
        # Regular headers: **kana【kanji】**[V1＋V2]...
        is_header = False
        parsed_hdr = None
        
        if line_strip.startswith('**'):
            # exclude bold noun lines or bold adverb lines
            if not (line_strip.startswith('**名詞') or line_strip.startswith('**(あちこち') or line_strip.startswith('**相撲用語')):
                parsed_hdr = parse_header_components(line)
                if parsed_hdr:
                    is_header = True
                    
        if is_header:
            if current_entry:
                entries.append(current_entry)
                
            current_entry = {
                'headword': line_strip,
                'kana': parsed_hdr['kana'],
                'kanji': parsed_hdr['kanji'],
                'v1_v2': parsed_hdr['v1_v2'],
                'is_redirect': parsed_hdr['is_redirect'],
                'redirect_target': parsed_hdr['redirect_target'],
                'sections': []
            }
            current_section_type = 'explanation'
            
            # If regular entry and has text after the header block, start the explanation section
            if not parsed_hdr['is_redirect'] and parsed_hdr['explanation_start']:
                current_entry['sections'].append({
                    'type': 'explanation',
                    'content': [parsed_hdr['explanation_start']]
                })
        else:
            if current_entry:
                if not line_strip:
                    # Skip empty lines to keep database clean
                    continue
                    
                # Check for page marker
                if re.match(r'^<\d+>$', line_strip):
                    current_entry['sections'].append({
                        'type': 'page_marker',
                        'content': [line_strip]
                    })
                    continue
                    
                # Clean line of outer ** for checking section type
                line_clean = line_strip
                if line_clean.startswith('**') and line_clean.endswith('**'):
                    line_clean = line_clean[2:-2].strip()
                elif line_clean.startswith('**'):
                    line_clean = line_clean[2:].strip()
                    
                # Check for section markers
                if line_clean.startswith('【副⁺】') or line_clean.startswith('【副】') or line_clean.startswith('副(') or line_clean.startswith('副（'):
                    current_section_type = 'adverb'
                    current_entry['sections'].append({
                        'type': 'adverb',
                        'content': [line_strip]
                    })
                elif line_clean.startswith('【例文】') or line_clean.startswith('例文') or line_clean.startswith('【例 文】'):
                    # Noun examples vs Verb examples
                    # Check if there is an active noun section in this entry
                    has_noun_section = any(s['type'] == 'noun' for s in current_entry['sections'])
                    if has_noun_section:
                        current_section_type = 'noun_examples'
                    else:
                        current_section_type = 'verb_examples'
                    current_entry['sections'].append({
                        'type': current_section_type,
                        'content': [line_strip]
                    })
                elif line_clean.startswith('【名詞形】') or line_clean.startswith('名詞形') or line_clean.startswith('【名詞】') or line_clean.startswith('名詞'):
                    current_section_type = 'noun'
                    current_entry['sections'].append({
                        'type': 'noun',
                        'content': [line_strip]
                    })
                elif line_clean.startswith('【補足】') or line_clean.startswith('補足') or line_clean.startswith('【動】'):
                    current_section_type = 'other'
                    current_entry['sections'].append({
                        'type': 'other',
                        'content': [line_strip]
                    })
                else:
                    # Continue current section
                    # If there are no sections yet, create explanation section
                    if not current_entry['sections']:
                        current_entry['sections'].append({
                            'type': 'explanation',
                            'content': [line_strip]
                        })
                    else:
                        # Append to the last section of the same type if it is active,
                        # or append to the last section if it matches current_section_type.
                        # Usually we just append to the last section if its type matches current_section_type.
                        last_sec = current_entry['sections'][-1]
                        if last_sec['type'] == current_section_type:
                            last_sec['content'].append(line_strip)
                        else:
                            # Create new section of current_section_type
                            current_entry['sections'].append({
                                'type': current_section_type,
                                'content': [line_strip]
                            })
                            
    if current_entry:
        entries.append(current_entry)
        
    print(f"Parsed {len(entries)} entries into JSON structure.")
    
    # Second pass: convert flat section contents to structured senses
    sequence_errors = []
    
    for entry in entries:
        kana_list = entry.get('kana', [])
        kanji_list = entry.get('kanji', [])
        headword = f"{'/'.join(kana_list)}【{'/'.join(kanji_list)}】" if kanji_list else '/'.join(kana_list)
        
        for sec in entry['sections']:
            stype = sec['type']
            if stype == 'page_marker':
                continue
                
            # Merge content
            full_text = " ".join(sec['content']).strip()
            
            if stype == 'explanation':
                senses = parse_hierarchy_explanation(full_text, sequence_errors, headword)
                sec['senses'] = senses
                del sec['content']
            elif stype == 'noun':
                senses = parse_hierarchy_flat(full_text, sequence_errors, headword, stype)
                sec['senses'] = senses
                del sec['content']
            elif stype in ['adverb', 'other']:
                sec['senses'] = [{'num': '', 'text': full_text}]
                del sec['content']
            elif stype in ['verb_examples', 'noun_examples']:
                senses = parse_hierarchy_examples(full_text, sequence_errors, headword, stype)
                sec['senses'] = senses
                del sec['content']
                
    # Save sequence errors log
    seq_errors_file = r"d:\OCR Project\复合动词\sequence_errors.json"
    with open(seq_errors_file, 'w', encoding='utf-8') as f:
        json.dump(sequence_errors, f, ensure_ascii=False, indent=2)
    print(f"Saved {len(sequence_errors)} sequence validation errors to {seq_errors_file}.")
    
    # Save as JSON
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(entries, f, ensure_ascii=False, indent=2)
        
    print(f"Saved JSON database to {output_file}.")

if __name__ == '__main__':
    main()
