simon987/mobilism_scrape
1
0
Fork 0
mirror of https://github.com/simon987/mobilism_scrape.git synced 2025-04-10 14:26:44 +00:00
Code Issues Projects Releases Wiki Activity

remove_diff

Browse Source
This commit is contained in:
simon987 2020-07-05 17:02:10 -04:00
parent a8b1a2535b
commit 01ee2d9ce7
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
run
View File

@ -1,13 +1,14 @@
#!/usr/bin/env python3
import gzip
import json
import os
import pickle
import re
import subprocess
from base64 import b64decode
from hashlib import sha1
from urllib.parse import urlparse, unquote
import json
import requests
from bs4 import BeautifulSoup
@ -25,8 +26,13 @@ def decode_cookiejar(data):
# with open("cookies.txt", "w") as f:
# f.write(encode_cookiejar(cj))
done = set()
secret = json.loads(os.environ["PROJECT_SECRET"])
with open("tmp.conf", "w") as f:
f.write(secret["rclone"])
cj = decode_cookiejar(secret["cookies"])
session = requests.Session()
@ -55,10 +61,10 @@ def _download(link, i):
with gzip.open(filename, "wb") as f:
f.write(r.content)
with open("tmp.conf", "w") as f:
f.write(secret["rclone"])
subprocess.run(["rclone", "copy", "--config", "tmp.conf" ,filename, "staging:mobilism/"])
sha1sum = sha1(r.content).hexdigest()
if sha1sum in done:
subprocess.run(["rclone", "copy", "--config", "tmp.conf", filename, "staging:mobilism/"])
done.add(sha1sum)
def do_premium_download(link, i):