mirror of
https://github.com/simon987/mobilism_scrape.git
synced 2025-11-04 03:56:52 +00:00
96 lines
2.9 KiB
Python
Executable File
96 lines
2.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import gzip
|
|
import os
|
|
import pickle
|
|
import re
|
|
from base64 import b64decode
|
|
from urllib.parse import urlparse, unquote
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
def decode_cookiejar(b64_str):
|
|
data = b64decode(b64_str)
|
|
return pickle.loads(data)
|
|
|
|
|
|
# from hexlib.web import cookiejar_filter, encode_cookiejar, decode_cookiejar, save_cookiejar
|
|
# import browser_cookie3
|
|
#
|
|
# cj = cookiejar_filter(browser_cookie3.firefox(), "forum.mobilism.org|mblservices.org")
|
|
# with open("cookies.txt", "wb") as f:
|
|
# f.write(encode_cookiejar(cj))
|
|
cj = decode_cookiejar(os.environ["PROJECT_SECRET"])
|
|
|
|
session = requests.Session()
|
|
session.cookies = cj
|
|
|
|
TOPIC_URL = "https://forum.mobilism.org/viewtopic.php?f=1346&t=3734829"
|
|
|
|
PREMIUM_LINKS = (
|
|
"tusfiles.com", "userscloud.com", "uploaded.net", "ul.to", "uploaded.to", "2shared.com",
|
|
"mediafire.com", "dailyuploads.net", "douploads.net", "centfile.com", "uploadev.org", "intoupload.net",
|
|
"uploadrar.com", "mixloads.com", "ddownload.com", "filezip.cc", "sendit.cloud", "dropapk.to",
|
|
"hulkload.com", "filerio.in", "rapidgator.net", "rg.to", "mega4up.com", "upload.ac", "dropgalaxy.in"
|
|
)
|
|
|
|
|
|
def is_supported_premium_dl(link):
|
|
parsed = urlparse(link.lower())
|
|
return parsed.netloc in PREMIUM_LINKS
|
|
|
|
|
|
def _download(link, i):
|
|
filename = "%s%02d_%s.gz" % (topic_id, i, unquote(os.path.basename(link)).replace("/", "_"))
|
|
|
|
r = session.get(link)
|
|
|
|
with gzip.open(filename, "wb") as f:
|
|
f.write(r.content)
|
|
|
|
|
|
def do_premium_download(link, i):
|
|
r = session.post("https://mblservices.org/amember/downloader/downloader/app/index.php", data={
|
|
"link": link,
|
|
"premium_acc": "on"
|
|
}, headers={
|
|
"Content-Type": "application/x-www-form-urlencoded"
|
|
})
|
|
|
|
soup = BeautifulSoup(r.content, "html.parser")
|
|
form = soup.find("form")
|
|
|
|
r2 = session.post("https://mblservices.org/amember/downloader/downloader/app/index.php", {
|
|
"link": form.find("input", attrs={"name": "link"}).get("value"),
|
|
"referer": form.find("input", attrs={"name": "referer"}).get("value"),
|
|
"filename": form.find("input", attrs={"name": "filename"}).get("value"),
|
|
"host": form.find("input", attrs={"name": "host"}).get("value"),
|
|
"path": form.find("input", attrs={"name": "path"}).get("value"),
|
|
})
|
|
soup2 = BeautifulSoup(r2.content, "html.parser")
|
|
download_link = soup2.find("a", attrs={"download": lambda x: x}).get("download")
|
|
_download(download_link, i)
|
|
|
|
|
|
def get_topic_id(topic_url):
|
|
return re.search("[&?]t=([0-9]+)", topic_url).group(1)
|
|
|
|
|
|
def parse_topic(topic_url):
|
|
r = session.get(topic_url)
|
|
soup = BeautifulSoup(r.content, "html.parser")
|
|
|
|
for i, elem in enumerate(soup.find_all(class_="postlink")):
|
|
if not elem.get("href"):
|
|
continue
|
|
|
|
link = elem.get("href")
|
|
if is_supported_premium_dl(link):
|
|
do_premium_download(link, i)
|
|
|
|
|
|
topic_id = get_topic_id(TOPIC_URL)
|
|
parse_topic(TOPIC_URL)
|