import re
from bs4 import BeautifulSoup
import string
import json


def compse(dst, sdt):

    translator = str.maketrans('', '', string.punctuation)
    dst = dst.translate(translator).strip()
    sdt = sdt.translate(translator).strip()

    dst = dst.replace('said of', 'of')
    sdt = sdt.replace('said of', 'of')

    if dst == sdt:
        return 1
    else:
        return 0

with open("repl.json", "r") as frr:
    reps = json.load(frr)

with open("labels.json", "r") as frr:
    labs = set(json.load(frr))



hds = dict()
with open("test.txt", "r") as frt:
    tet = frt.read()
    for rep in reps:
        tet = tet.replace(rep['bfr'], rep['aft'])
    tet = re.sub(r' {2,}', ' ', tet)

with open("ndc.txt", "w") as fwn:
    fwn.write(tet)

with open("ndc.txt", "r") as frn:
    for line in frn:
        line = line.strip()

        if '<div class="eBody"><p>' in line:
            continue

        line = line.replace('<i><span class="bodyi">v aux</span></i>', '')

        line = line.replace(">‘s", ">'s").replace("</span>‘", "</span>’")

        line = line.replace('.<a ', '</div><div class="sect2"><a ')

        athes = {'athe': 'the', 'anthe': 'the', 'thise': 'the', 'hasve': 'have'}
        for at, he in athes.items():
            line = re.sub(fr'\b{at}\b', fr'{he}', line)

        line = re.sub(r'\b(?=<a )', ' ', line)

        line = re.sub(r' (?=[.,;:])', '', line)

        line = re.sub(r'(?<=[,;:+])(?=<a |<span |[a-zA-Z])', ' ', line)

        line = re.sub(r'(</a>|</span>|[a-zA-Z])\+', r'\1 +', line)

        line = line.replace('.</p>', '</p>')
        line = line.replace('.</div>', '</div>')

        line = line.replace(']]', ']')
        line = line.replace(')<i>', ') <i>')

        line = line.replace('<br/>', '┳')

        line = line.replace('<div class="sidebar">', '</div><div class="sbtmp"><div class="sidebar">')


        line = re.sub(r'<i></i></p><p>([^<>]*)</p>', r'<i><p>\1</p></i></p>', line)

        line = re.sub(r'<i><p>([^<>]*)</p></i>', r'<span class="eg">\1</span>', line)
        
        line = re.sub(r'(?<!<p>)(<span class="eg">)', r'</p> <p class="egp">\1', line)

        line = line.replace('<p><span class="eg">', '<p class="egp"><span class="eg">')
        

        line = re.sub(r'<span class="bodyi">(<span class="bodyi">[^<>]*</span>)</span>', r'\1', line)

        line = re.sub(r'(<i><span class="bodyi">)(/[^<>/]*/) *', r'\2 \1', line)

        line = re.sub(r'\B(/(?:<span class="bodyi">[^<>/]*</span>|[^<>/])+/)\B', r'<pron>\1</pron>', line)

        line = re.sub(r'<i><span class="bodyi">([^<>]*)</span></i>', r'<span class="pos">\1</span>', line)

        line = re.sub(r'(<span class="bodyi">(?:[^<>()]*[^ <>()])?) *\(([^()]*?</span>)\)', r'\1</span> (<span class="bodyi">\2)', line)
        

        line = re.sub(r'(<p class="egp"><span class="eg">[^<>]*</span></p>) \1', r'\1', line)

        line = re.sub(r'(\([^<>()]*)(</span>)\)', r'\1)\2', line)
        line = line.replace('</span>(', '</span> (')

        line = re.sub(r'(?<=[^()]{3}\))(?=\w)', 'ノ ', line)
        line = line.replace('</sub>)ノ ', '</sub>)')
        line = line.replace('ノ', '')


        line = re.sub(r'(<span class="bodyi">(?:esp|collectively)</span>)(?=\w)', r'\1 ', line)
        line = re.sub(r'(?<=\w)(?=<span class="bodyi">[^<>]*</span>\W)', ' ', line)



        soup0 = BeautifulSoup(line, 'html5lib')

        sec0 = soup0.body
        sec0.name = 'entry'

        dmt0 = sec0.select_one('h3 span.page-heading')
        if not dmt0:
            continue

        hw0 = list(dmt0.stripped_strings)[0]
        if not hw0:
            continue

        for ass in sec0.select('a[name],span[class^="sHead"]'):
            ass.decompose()

        for bibi in sec0.select('.bodyi .bodyi,.sidebar>p,div.sbtmp'):
            bibi.unwrap()


        while sec0.select_one('*:empty'):
            emp = sec0.select_one('*:empty')
            emp.decompose()

        dctmp = str(sec0)
        dctmp = re.sub(r' {2,}', ' ', dctmp)
        
        dctmp = re.sub(r'(\[[^][]+\]) ?((?:</p>)?</div>)', r'\2<div class="etym">\1</div>', dctmp)
        
        dctmp = dctmp.replace('┳<p>', '<p class="ssp">')
        dctmp = dctmp.replace('┳', '')
        dctmp = dctmp.replace('<div class="sect1"><p>', '<div class="sect1"><p class="ssp">')
        dctmp = dctmp.replace('<p class="ssp"> ', '<p class="ssp">')
        
        dctmp = dctmp.replace('</p><p> ', 'ಠ — ')
        dctmp = re.sub(r'(</p>)( — [^<>✱]*)', r'ಠ\2\1', dctmp)
        dctmp = dctmp.replace('— —', '—')
        
        dctmp = re.sub(r'(?<=</p> )<p>(?=\w)', r'</div><div class="sect2"><p class="ssp">', dctmp)


        dctmp = dctmp.replace('</span>/<span class="bodyb">', '/')

        dctmp = re.sub(r' *✱ (<span class="bodyb">[^<>]+</span>) *', r'</p></div><div class="phr">\1</div><div class="sect2"><p class="ssp">', dctmp)

        dctmp = dctmp.replace('<div class="sect1"><p class="ssp"></p></div>', '<div class="sect1">')

        dctmp = dctmp.replace('</pron>', 'ᗜ</pron>')

        dctmp = re.sub(r' *(<span class="bodyb">[^<>]+</span> (?:<pron>[^ᗜ]*ᗜ</pron> |\([^()]*?\) )*<span class="pos">[^<>]+</span>) *', r'</p></div><div class="deriv">\1<p>', dctmp)

        dctmp = dctmp.replace('ᗜ', '')
        

        excs = {
                  "Conf\u00e9d\u00e9ration Helv\u00e9tique",
                  "often",
                  "treated as sing. or pl",
                  "train grande vitesse",
                  }

        labs = labs - excs
        for lab in labs:
            dctmp = dctmp.replace(f'<span class="bodyi">{lab}</span>', f'<span class="label">{lab}</span> ')


        dctmp = re.sub(r'(<div class="sect2"><p class="ssp">)(<span class="pos">[^<>]*</span>) *', r'\2\1', dctmp)

        dctmp = re.sub(r'(<span class="pos">[^<>]*</span>(?: <span class="label">[^<>]*</span>)?) *((?:</p>)?(?:</div>)+)', r'\2\1', dctmp)


        dctmp = re.sub(r'(<span class="bodysc">[^<>]+) (\d+)(</span>)', r'\1<sup>\2</sup>\3', dctmp)

        
        soup1 = BeautifulSoup(dctmp, 'html5lib')
        sec1 = soup1.entry

        while sec1.select_one('*:empty'):
            emp = sec1.select_one('*:empty')
            emp.decompose()
        
        while sec1.select_one('div.sect2+div.sect3'):
            dss = sec1.select_one('div.sect2+div.sect3')
            dssp = dss.find_previous_sibling('div')
            dsse = dss.extract()
            dssp.append(dsse)
        
        while sec1.select_one('div>p:last-child:not([class])'):
            dpn = sec1.select_one('div>p:last-child:not([class])')
            dpnt = dpn.get_text().strip()
            dpn.name = 'div'
            if '—' in dpnt:
                dpn['class'] = 'enote'
            else:
                dpn['class'] = 'whis'
            dpnp = dpn.parent
            dpne = dpn.extract()

            dnpt = dpnp.get_text().strip()
            if dpnt not in dnpt:
                dpnp.insert_after(dpne)
        
        dctmp = str(sec1)
        dctmp = re.sub(r' {2,}', ' ', dctmp)

        dctmp = re.sub(r'(?<!<div class="etym">)(\[[^][]+\]) ?(</p>)', r'\2<span class="etym">\1</span>', dctmp)

        dctmp = dctmp.replace('</p><p>', ' ')
        

        soup = BeautifulSoup(dctmp, 'html5lib')
        sec = soup.entry

        while sec.select_one('*:empty'):
            emp = sec.select_one('*:empty')
            emp.decompose()
        
        while sec.select_one('div.sect2>div:last-child:not(div.sect3)'):
            dpl = sec.select_one('div.sect2>div:last-child:not(div.sect3)')
            dplp = dpl.parent
            dple = dpl.extract()
            dplp.insert_after(dple)
        
        for dds0 in sec.select('div.phr+div.sect2'):
            dds0p = dds0.find_previous_sibling('div')
            dds0.name = 'ptmp'
            dds0e = dds0.extract()
            dds0p.append(' ')
            dds0p.append(dds0e)
        
        fla = sec.find('div', class_='flag')
        if fla:
            for sib in fla.find_previous_siblings('div'):
                sib.decompose()
            fla.decompose()
        
        gal = sec.find('div', class_='galf')
        if gal:
            for sib in gal.find_next_siblings('div'):
                sib.decompose()
            gal.decompose()

        for disc in sec(class_='discard'):
            disc.decompose()
        

        heps = sec.select('h3 .pos')
        art = sec.article
        if not art:
            continue
        spvs = art('span', class_='pos', string=re.compile('^verb '))
        if not heps or not spvs:
            pass
        elif heps[-1].get_text().strip() == 'verb' and len(spvs) > 1:
            for sib in spvs[0].find_previous_siblings('div'):
                sib.decompose()
        

        for dpdp in sec.select('div.deriv'):
            dpt = dpdp.get_text().strip()
            dpco = 0
            for pdpd in sec.select('div.deriv'):
                pdt = pdpd.get_text().strip()
                if pdt == dpt:
                    dpco += 1
            if dpco > 1:
                dpdp.decompose()

        while sec.select_one('div.phr+div.sect2'):
            dds = sec.select_one('div.phr+div.sect2')
            ddsp = dds.find_previous_sibling('div')
            dds.name = 'ptmp'
            ddse = dds.extract()
            ddsp.append(' ')
            ddsp.append(ddse)

        for dpot in sec.select('div.phr>ptmp:only-of-type'):
            dpot.unwrap()

        for dppt in sec.select('div.phr>ptmp'):
            dppt.name = 'div'


        for st2 in sec.select('.sect2:not(.phr *)'):
            if st2(class_='sect3'):
                continue
            st2p = st2.find(class_='ssp')
            sco = 0
            for st2n in st2.find_next_siblings():
                if 'pos' in st2n.get('class'):
                    break
                elif 'sect2' not in st2n.get('class'):
                    continue
                elif not st2n(class_='sect3'):
                    continue
                for stnp in st2n(class_='ssp'):
                    sco += compse(st2p.get_text(), stnp.get_text())
                if sco:
                    st2.decompose()
                    break
        
        for st3 in sec.select('.sect2>.sect3:not(.phr *)'):
            st3p = st3.find(class_='ssp')
            sco = 0
            for st3n in st3.parent.find_next_siblings():
                if 'pos' in st3n.get('class'):
                    break
                elif 'sect2' not in st3n.get('class'):
                    continue
                for stnp in st3n(class_='ssp'):
                    sco += compse(st3p.get_text(), stnp.get_text())
                if sco:
                    st3['class'].append('error')
                    break

        
        dds = sec.select('div.deriv~div.sect2')
        if dds:
            ddsl = dds[-1]
            for ddsp in ddsl.find_previous_siblings('div', class_='deriv'):
                ddse = ddsp.extract()
                ddsl.insert_after(ddse)
        
        dds = sec.select('div.phr~div.sect2')
        if dds:
            ddsl = dds[-1]
            for ddsp in ddsl.find_previous_siblings('div', class_='phr'):
                ddse = ddsp.extract()
                ddsl.insert_after(ddse)

        for holder in sec('span', class_='placeholder', string='PLACEHOLDER'):
            holder.decompose()

        
        for st3 in sec.select('.sect2+.sect2>.sect3:not(.sect3~.sect3)'):
            stp = st3.parent
            stpp = stp.find_previous_sibling(class_='sect2')
            if stpp.find(class_='sect3'):
                continue
            
            st3.find('p', class_='ssp').insert_before('ﾐ')
            st3.find('p', class_='ssp').insert_after('ﾐ')
            stts = [stt.strip() for stt in stp.get_text().split('ﾐ')]
            st3t = ' '.join(stts[:2]).strip()
            stpt = stpp.find('p', class_='ssp').get_text().strip()

            if st3t.startswith(stpt) or stpt.startswith(st3t):
                stpp.decompose()


        for acb in sec.select('a .crossRef .bodysc'):
            ahr = list(acb.stripped_strings)[0]
            acbp = acb.find_parent('a')
            acbp.attrs.clear()
            acbp['href'] = f'entry://{ahr}'

        for egp in sec.select('.egp'):
            egpt = egp.get_text().strip()
            if 'ಠ' in egpt:
                egp['class'] = 'quote'


        for dpdp in sec.select('div.phr'):
            dpsb = dpdp.find('span', class_='bodyb')
            hwdp = dpsb.get_text().strip()
            dme = f'<div class="me">Main Entry: <a href="entry://{hw0}">{hw0}</a></div>'
            dcdp = f'<subentry>{str(dpdp)}{dme}</subentry>'
            dcdp = dcdp.replace('ಠ', '').replace('ﾐ', '')
            dcdp = re.sub(r' {2,}', ' ', dcdp)

            hds.setdefault(hwdp, set()).add(dcdp)
            print(hwdp)

        
        dc0 = str(sec)
        dc0 = dc0.replace('ಠ', '').replace('ﾐ', '')
        dc0 = re.sub(r' {2,}', ' ', dc0)

        hds.setdefault(hw0, set()).add(dc0)
        print(hw0)


CSS = '<link href="tped.css" rel="stylesheet">'

khds = set()
for hw, dcs in hds.items():
    kk = hw.lower()
    for spu in string.punctuation:
        kk = kk.replace(spu, 'zzzzzz')
    dc = '<hr>'.join(sorted(dcs))
    khd = (kk, hw, dc)
    khds.add(khd)

for kk, hw, dc in sorted(khds):
    with open("dict.txt", "a") as fad:
        fad.write(f'{hw}\n{CSS}{dc}\n</>\n')


