import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urldefrag
from multiprocessing import Pool


def get_rtext(url):
    while True:
        try:
            r = requests.get(url, allow_redirects=False, timeout=5)
        except:
            print('ERROR')
        else:
            if r.status_code == 200:
                break
            elif r.status_code == 500:
                url = url.split('<')[0]
            else:
                log = f'{r.status_code} | {url}'
                print(log)

    return r.text


def get_dc(soup):
    dc = ''
    des = soup.select('div.entryRH')

    for de in des:
        dc += str(de)

    dc = dc.replace('\n', '')
    return dc


def iter_m(nmset_u):
    nm, set_u = nmset_u
    set_hr = set()

    for url in set_u:
        rt = get_rtext(url)
        soup = BeautifulSoup(rt, 'html5lib')

        for ah in soup.select('a[href]'):
            base = 'https://www.wordreference.com/definition/'
            hr = urldefrag(urljoin(base, ah['href'])).url
            if hr.startswith(base):
                set_hr.add(hr)
                print(hr)

        dc = get_dc(soup)
        if dc:
            dnt = f'dc_{nm}.txt'
            with open(dnt, encoding='utf-8', mode='a') as fad:
                fad.write(f'{dc}\n')

    return set_hr


def wget_m(set_u, set_d):
    N = 8
    sets_u = [set() for i in range(N)]

    for number, url in enumerate(set_u):
        sets_u[number%N].add(url)

    with Pool(N) as p:
        sets_hr = p.map(iter_m, enumerate(sets_u))

    set_hr = set().union(*sets_hr)

    set_u = set_hr - set_d
    set_d = set_hr | set_d

    if set_u:
        wget_m(set_u, set_d)

if __name__ == '__main__':
    seed = {'https://www.wordreference.com/definition/accord'}
    wget_m(seed, seed)