|
@@ -13,29 +13,26 @@ def scrape(url, arg=None):
|
|
|
else:
|
|
|
data = requests.get(f"{url}{arg}")
|
|
|
|
|
|
- if data.status_code == 200:
|
|
|
- our_path = re.sub(r".*://.*/", "/", request.url)
|
|
|
- path = re.sub(r".*://.*/", "/", data.url)
|
|
|
- print(our_path, path)
|
|
|
- if our_path != path:
|
|
|
- return f"REDIRECT {path}"
|
|
|
- ret = []
|
|
|
- soup = BeautifulSoup(data.text, "html.parser")
|
|
|
- for div in soup.find_all("div"):
|
|
|
- defid = div.get('data-defid')
|
|
|
- if defid != None:
|
|
|
- definition = soup.find(attrs={"data-defid": [defid]})
|
|
|
- word = definition.select("div div h1 a, div div h2 a")[0].text
|
|
|
- meaning = definition.find(attrs={"class" : ["break-words meaning mb-4"]}).decode_contents()
|
|
|
- example = definition.find(attrs={"class" : ["break-words example italic mb-4"]}).decode_contents()
|
|
|
- contributor = definition.find(attrs={"class" : ["contributor font-bold"]})
|
|
|
- ret.append([defid, word, meaning, example, contributor])
|
|
|
- pages = soup.find(attrs={"class" : ["pagination text-xl text-center"]})
|
|
|
- if pages == None:
|
|
|
- pages = ""
|
|
|
- return (ret, pages)
|
|
|
- else:
|
|
|
- return f"Couldn't get data from Urban Dictionary\n{data.status_code}"
|
|
|
+ our_path = re.sub(r".*://.*/", "/", request.url)
|
|
|
+ path = re.sub(r".*://.*/", "/", data.url)
|
|
|
+ print(our_path, path)
|
|
|
+ if our_path != path:
|
|
|
+ return f"REDIRECT {path}"
|
|
|
+ ret = []
|
|
|
+ soup = BeautifulSoup(data.text, "html.parser")
|
|
|
+ for div in soup.find_all("div"):
|
|
|
+ defid = div.get('data-defid')
|
|
|
+ if defid != None:
|
|
|
+ definition = soup.find(attrs={"data-defid": [defid]})
|
|
|
+ word = definition.select("div div h1 a, div div h2 a")[0].text
|
|
|
+ meaning = definition.find(attrs={"class" : ["break-words meaning mb-4"]}).decode_contents()
|
|
|
+ example = definition.find(attrs={"class" : ["break-words example italic mb-4"]}).decode_contents()
|
|
|
+ contributor = definition.find(attrs={"class" : ["contributor font-bold"]})
|
|
|
+ ret.append([defid, word, meaning, example, contributor])
|
|
|
+ pages = soup.find(attrs={"class" : ["pagination text-xl text-center"]})
|
|
|
+ if pages == None:
|
|
|
+ pages = ""
|
|
|
+ return (ret, pages)
|
|
|
|
|
|
def render(data):
|
|
|
return render_template('index.html', data=data)
|