from bs4 import BeautifulSoup
import re
from multiprocessing import Pool
from unidecode import unidecode


def clean_ex(matchobj):
    return matchobj.group(0).replace('</span><span class="rh_ex">', ' ')


def dc_repl(dc):
    dc = dc.replace(' ', ' ').replace(":''", '":').replace("''", '"')
    dc = re.sub(r' {2,}', r' ', dc)

    dc = dc.replace('( ', '(').replace('[ ', '[')
    dc = dc.replace(' )', ')').replace(' ]', ']')

    dc = dc.replace('toerrishuman</span>.', '.</span>').replace('toerrishuman', '')

    dc = dc.replace('-<sc>', '<sc>-').replace('</sc>-<sup>', '-</sc><sup>')
    dc = re.sub(r'(?<!<xr>)(See|Compare) ?(<sc>[^<>]+</sc>(?:<sc>[^<>]+</sc>|[^<>]|<sup>\d+</sup>)*\.|<targ><sc>.+?</targ>)', r'<xr>\1 \2</xr>', dc)

    dc = re.sub(r'([,;]\W*)(</b>)', r'\2\1', dc)

    dc = re.sub(r'(<span>\]</span></span>)(\)?)\]', r'\2\1', dc)

    dc = re.sub(r'(?<!<div class="rh_main">)(<span class="rh_me">)', r'<hr>\1', dc)

    dc = re.sub(r' <span class="rh_txt">[][]:?</span> ', r' ', dc)

    dc = re.sub(r'<span>\]</span></span>( <span class="rh_txt">\+</span> )(?:<span class="rh_lab"><span>\[</span>)?', r'\1', dc)


    dc = re.sub(r'\[([^][]*)(<span class="rh_lab"><span>\[</span>)', r'\2\1', dc)


    dc = re.sub(r'(?<!<span>)(\[[^][]*<[^][<>]+>[^][]*\])', r'<span class="rh_lab">\1</span>', dc)

    dc = re.sub(r'\[[^][]*</span><span class="rh_ex">[^][]*\]', clean_ex, dc)
    dc = re.sub(r'\([^)(]*</span><span class="rh_ex">[^)(]*\)', clean_ex, dc)

    
    '''
    dc = re.sub(r'(\]</span></span>)(</span>)(<span class="rh_ex">[^<>]*</span>|\w[^<>]*(?:<span class="rh_ex">)?[^<>]*</span>)', r'\1\3\2', dc)
    '''

    dc = dc.replace(']:<span class="rh_ex">', '<span>]</span></span><span class="rh_ex">')

    dc = dc.replace('</i></span><i>', '</i></span><span class="rh_ex"><i>')

    
    dc = dc.replace('<span class="rh_empos">', '<br><span class="rh_empos">')

    dc = re.sub(r': ?<span class="rh_ex">', r'<span class="rh_ex">: ', dc)

    dc = re.sub(r'(</b>)(<b>′</b>[^,.]*[,.])', r'\2\1', dc)

    return dc


def dc_soup(dc):
    soup1 = BeautifulSoup(dc, 'html5lib')
    s1b = soup1.select_one('div.rh_main')

    for ell in s1b.select('.rh_ex > .rh_lab:last-child'):
        if ell.parent.contents[-1] != ell:
            continue

        for els in ell('span', string=re.compile(r'^[][]$')):
            els.decompose()
        ell.unwrap()

    for lai in s1b.select('.rh_lab :not(sup):not(sub)'):
        lai.unwrap()
    
    for rb in s1b.select('ros br,def + br'):
        rb.decompose()

    for bb in s1b.select('b b,supnt xr'):
        bb.unwrap()

    dfs = ['sdef', 'def']
    for df in dfs:
        for de in s1b.select(f'{df} + .rh_ex'):
            dps = de.find_previous_sibling(df)
            dee = de.extract()
            dps.append(dee)

    for ded in s1b.select('def > .rh_ex + def'):
        ded.insert_before('hodor')

    dc = str(s1b).replace('hodor', '</def>')
    dc = dc.replace('<span class="rh_ex">: ', ': <span class="rh_ex">')

    return dc


def dc_entr(dc):
    soup2 = BeautifulSoup(dc, 'html5lib')
    s2b = soup2.body
    s2b.name = 'entry'
    dc = str(s2b)

    return dc


def get_hw(soup):
    for sup in soup.select('sup,em.rh_pron,.rh_me var'):
        sup.decompose()

    hw = soup.get_text().strip()

    hw = hw.replace('•', '').replace('ˌ', '').replace('ˈ', '').replace('′', '').strip('\'-,;:!? ')

    return hw


def get_hws(dc):
    soup = BeautifulSoup(dc, 'html5lib')
    for rmva in soup.select('.rh_me + var'):
        rm = rmva.find_previous_sibling(class_='rh_me')
        if rmva.get_text().startswith('('):
            tmp = get_hw(rm) + get_hw(rmva)
            tmp = re.sub(r'\([^)(]*\)', r'', tmp)
            hws = tmp.split(' or ')
            return hws

    hws = list()
    for rm in soup.select('.rh_me'):
        hw = get_hw(rm)
        if hw:
            hws.append(hw)
            hws.append(unidecode(hw, errors="preserve"))

    for vb in soup.select('var b'):
        if vb.get_text().startswith('-'):
            continue
        hw = get_hw(vb).strip('. ')
        if hw:
            hws.append(hw)

    return hws


def dedup(hws):
    nhws = list()
    tmps = list()

    for hw in hws:
        tmp = re.sub(r'\W', r'', hw).lower()
        if tmp in tmps:
            continue
        nhws.append(hw)
        tmps.append(tmp)

    return nhws


def extract_hd(lns):
    hds = set()
    for ln in lns:
        soup = BeautifulSoup(ln, 'html5lib')
        for de in soup.select('div.entryRH'):
            ad = de.select_one('[id^="advanced_"]')
            if not ad:
                continue

            for al in de.select('a,ol,ul,li,lab,i .rh_ex'):
                al.unwrap()

            for su in de('span', string='USA pronunciation'):
                su.decompose()

            for ae in de.select('art,br,:empty'):
                ae.decompose()

            for st in de.select('[style]'):
                del st['style']

            dfs = ['pdef', 'def', 'sdef', 'ros', 'xr', 'var', 'sc']
            for df in dfs:
                for ddf in de(class_=f'rh_{df}'):
                    ddf.attrs.clear()
                    ddf.name = df

            for us in de(class_='supnt'):
                us.attrs.clear()
                us.name = 'supnt'

            for sc2 in de.select('sc > sc'):
                sc2.parent.unwrap()

            for rex in de.select('span.rh_ex'):
                rex.append('toerrishuman')

            de.attrs.clear()
            de['class'] = 'rh_main'

            dc1 = dc_repl(str(de))
            dc1 = dc_soup(dc1)
            dc1 = dc_entr(dc1)

            hw1s = dedup(get_hws(dc1))

            for hw1 in hw1s:
                hd1 = (hw1, dc1)
                hds.add(hd1)

            soup_dc1 = BeautifulSoup(dc1, 'html5lib')
            for srb in soup_dc1.select('def > b:only-of-type'):

                try:
                    hw2 = srb.contents[0].strip(',.;:!? ')
                except:
                    print(str(srb))
                    continue

                if not hw2 or hw2.startswith('—'):
                    continue

                srd = srb.parent
                dc2 = dc_entr(str(srd))

                if not re.match(r'<entry><def>(?: |<span class="rh_cat">[^<>]*</span>)*<b>', dc2):
                    continue

                hd2 = (hw2, dc2)
                hds.add(hd2)
        
    return hds

if __name__ == '__main__':
    with open('dc.txt', 'r') as frd:
        lns = {line.strip() for line in frd}

    N = 8
    sets_ln = [set() for i in range(N)]

    for number, ln in enumerate(lns):
        sets_ln[number%N].add(ln)                    

    with Pool(N) as p:
        sets_hd = p.map(extract_hd, sets_ln)

    hds = set().union(*sets_hd)

    for hd in sorted(hds):
        hw, dc = hd
        CSS = '<link rel="stylesheet" href="rhwdae.css">'
        with open('dict.txt', 'a') as fad:
            fad.write(f'{hw}\n{CSS}{dc}\n</>\n')
        print(hw)