#!/usr/bin/env python3 import gzip import os import pickle import re from base64 import b64decode from urllib.parse import urlparse, unquote import requests from bs4 import BeautifulSoup def decode_cookiejar(b64_str): data = b64decode(b64_str) return pickle.loads(data) # from hexlib.web import cookiejar_filter, encode_cookiejar, decode_cookiejar, save_cookiejar # import browser_cookie3 # # cj = cookiejar_filter(browser_cookie3.firefox(), "forum.mobilism.org|mblservices.org") # with open("cookies.txt", "wb") as f: # f.write(encode_cookiejar(cj)) cj = decode_cookiejar(os.environ["PROJECT_SECRET"]) session = requests.Session() session.cookies = cj TOPIC_URL = "https://forum.mobilism.org/viewtopic.php?f=1346&t=3734829" PREMIUM_LINKS = ( "tusfiles.com", "userscloud.com", "uploaded.net", "ul.to", "uploaded.to", "2shared.com", "mediafire.com", "dailyuploads.net", "douploads.net", "centfile.com", "uploadev.org", "intoupload.net", "uploadrar.com", "mixloads.com", "ddownload.com", "filezip.cc", "sendit.cloud", "dropapk.to", "hulkload.com", "filerio.in", "rapidgator.net", "rg.to", "mega4up.com", "upload.ac", "dropgalaxy.in" ) def is_supported_premium_dl(link): parsed = urlparse(link.lower()) return parsed.netloc in PREMIUM_LINKS def _download(link, i): filename = "%s%02d_%s.gz" % (topic_id, i, unquote(os.path.basename(link)).replace("/", "_")) r = session.get(link) with gzip.open(filename, "wb") as f: f.write(r.content) def do_premium_download(link, i): r = session.post("https://mblservices.org/amember/downloader/downloader/app/index.php", data={ "link": link, "premium_acc": "on" }, headers={ "Content-Type": "application/x-www-form-urlencoded" }) soup = BeautifulSoup(r.content, "html.parser") form = soup.find("form") r2 = session.post("https://mblservices.org/amember/downloader/downloader/app/index.php", { "link": form.find("input", attrs={"name": "link"}).get("value"), "referer": form.find("input", attrs={"name": "referer"}).get("value"), "filename": form.find("input", attrs={"name": "filename"}).get("value"), "host": form.find("input", attrs={"name": "host"}).get("value"), "path": form.find("input", attrs={"name": "path"}).get("value"), }) soup2 = BeautifulSoup(r2.content, "html.parser") download_link = soup2.find("a", attrs={"download": lambda x: x}).get("download") _download(download_link, i) def get_topic_id(topic_url): return re.search("[&?]t=([0-9]+)", topic_url).group(1) def parse_topic(topic_url): r = session.get(topic_url) soup = BeautifulSoup(r.content, "html.parser") for i, elem in enumerate(soup.find_all(class_="postlink")): if not elem.get("href"): continue link = elem.get("href") if is_supported_premium_dl(link): do_premium_download(link, i) topic_id = get_topic_id(TOPIC_URL) parse_topic(TOPIC_URL)