|
@@ -18,15 +18,27 @@ def scrape(url):
|
|
|
return f"REDIRECT {path}"
|
|
|
ret = []
|
|
|
soup = BeautifulSoup(data.text, "html.parser")
|
|
|
- for div in soup.find_all("div"):
|
|
|
- defid = div.get('data-defid')
|
|
|
- if defid != None:
|
|
|
- definition = soup.find(attrs={"data-defid": [defid]})
|
|
|
- word = definition.select("div div h1 a, div div h2 a")[0].text
|
|
|
- meaning = definition.find(attrs={"class" : ["break-words meaning mb-4"]}).decode_contents()
|
|
|
- example = definition.find(attrs={"class" : ["break-words example italic mb-4"]}).decode_contents()
|
|
|
- contributor = definition.find(attrs={"class" : ["contributor font-bold"]})
|
|
|
- ret.append([defid, word, meaning, example, contributor])
|
|
|
+
|
|
|
+ defs = [(div, div.get('data-defid')) for div in soup.find_all("div") if div.get('data-defid')]
|
|
|
+ try:
|
|
|
+ thumbs_data = {
|
|
|
+ str(entry['defid']): entry
|
|
|
+ for entry
|
|
|
+ in requests.get(
|
|
|
+ 'https://api.urbandictionary.com/v0/uncacheable?ids=' + ','.join(defid for (_, defid) in defs)
|
|
|
+ ).json()['thumbs']
|
|
|
+ }
|
|
|
+ except:
|
|
|
+ thumbs_data = {}
|
|
|
+
|
|
|
+ for (definition, defid) in defs:
|
|
|
+ word = definition.select("div div h1 a, div div h2 a")[0].text
|
|
|
+ meaning = definition.find(attrs={"class" : ["break-words meaning mb-4"]}).decode_contents()
|
|
|
+ example = definition.find(attrs={"class" : ["break-words example italic mb-4"]}).decode_contents()
|
|
|
+ contributor = definition.find(attrs={"class" : ["contributor font-bold"]})
|
|
|
+ thumbs_up = thumbs_data.get(defid, {}).get('up')
|
|
|
+ thumbs_down = thumbs_data.get(defid, {}).get('down')
|
|
|
+ ret.append([defid, word, meaning, example, contributor, thumbs_up, thumbs_down])
|
|
|
pages = soup.find(attrs={"class" : ["pagination text-xl text-center"]})
|
|
|
if pages == None:
|
|
|
pages = ""
|