rural-dict/main.py

#!/usr/bin/env python

from flask import Flask, render_template, request, redirect
import requests
import html
import re
from bs4 import BeautifulSoup
from urllib.parse import quote, unquote

def scrape(url):
    data = requests.get(url)
    
    our_path = re.sub(r".*://.*/", "/", request.url)
    path = re.sub(r".*://.*/", "/", data.url)
    if our_path != path and \
            quote(unquote(re.sub("[?&=]", "", our_path))) != re.sub("[?&=]", "", path):
                # this is bad ^
                return f"REDIRECT {path}"
    ret = []
    soup = BeautifulSoup(data.text, "html.parser")

    defs = [(div, div.get('data-defid')) for div in soup.find_all("div") if div.get('data-defid')]
    try:
        thumbs_data = {
            str(entry['defid']): entry
            for entry
            in requests.get(
                'https://api.urbandictionary.com/v0/uncacheable?ids=' + ','.join(defid for (_, defid) in defs)
            ).json()['thumbs']
        }
    except:
        thumbs_data = {}

    for (definition, defid) in defs:
        word = definition.select("div div h1 a, div div h2 a")[0].text
        meaning = definition.find(attrs={"class" : ["break-words meaning mb-4"]}).decode_contents()
        example = definition.find(attrs={"class" : ["break-words example italic mb-4"]}).decode_contents()
        contributor = definition.find(attrs={"class" : ["contributor font-bold"]})
        thumbs_up = thumbs_data.get(defid, {}).get('up')
        thumbs_down = thumbs_data.get(defid, {}).get('down')
        ret.append([defid, word, meaning, example, contributor, thumbs_up, thumbs_down])
    pages = soup.find(attrs={"class" : ["pagination text-xl text-center"]})
    if pages == None:
        pages = ""
    return (ret, pages)

app = Flask(__name__, template_folder="templates", static_folder="static")

@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def catch_all(path):
    scraped = scrape(f"https://urbandictionary.com/{re.sub(r'.*://.*/', '/', request.url)}")
    if type(scraped) == str and scraped.startswith("REDIRECT"):
        return redirect(scraped.replace("REDIRECT ", ""), 302)
    return render_template('index.html', data=scraped, term=request.args.get("term"))

if __name__ == '__main__':
    from waitress import serve
    serve(app, host="0.0.0.0", port=8080)