simon987/mobilism_scrape
1
0
Fork 0
mirror of https://github.com/simon987/mobilism_scrape.git synced 2025-04-19 18:46:46 +00:00
Code Issues Projects Releases Wiki Activity

remove_diff

Browse Source
This commit is contained in:
simon987 2020-07-05 17:02:10 -04:00
parent a8b1a2535b
commit 01ee2d9ce7
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
run
View File

@ -1,13 +1,14 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import gzip import gzip
import json
import os import os
import pickle import pickle
import re import re
import subprocess import subprocess
from base64 import b64decode from base64 import b64decode
from hashlib import sha1
from urllib.parse import urlparse, unquote from urllib.parse import urlparse, unquote
import json
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
@ -25,8 +26,13 @@ def decode_cookiejar(data):
# with open("cookies.txt", "w") as f: # with open("cookies.txt", "w") as f:
# f.write(encode_cookiejar(cj)) # f.write(encode_cookiejar(cj))
done = set()
secret = json.loads(os.environ["PROJECT_SECRET"]) secret = json.loads(os.environ["PROJECT_SECRET"])
with open("tmp.conf", "w") as f:
f.write(secret["rclone"])
cj = decode_cookiejar(secret["cookies"]) cj = decode_cookiejar(secret["cookies"])
session = requests.Session() session = requests.Session()
@ -55,10 +61,10 @@ def _download(link, i):
with gzip.open(filename, "wb") as f: with gzip.open(filename, "wb") as f:
f.write(r.content) f.write(r.content)
with open("tmp.conf", "w") as f: sha1sum = sha1(r.content).hexdigest()
f.write(secret["rclone"]) if sha1sum in done:
subprocess.run(["rclone", "copy", "--config", "tmp.conf", filename, "staging:mobilism/"])
subprocess.run(["rclone", "copy", "--config", "tmp.conf" ,filename, "staging:mobilism/"]) done.add(sha1sum)
def do_premium_download(link, i): def do_premium_download(link, i):