simon987/Misc-Download-Scripts
1
0
Fork 0
mirror of https://github.com/simon987/Misc-Download-Scripts.git synced 2025-04-10 06:16:43 +00:00
Code Issues Projects Releases Wiki Activity

Added progression and resume option

Browse Source
This commit is contained in:
simon987 2018-04-14 11:41:05 -04:00
parent 894c4f1328
commit 8212793dc5
1 changed files with 48 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
opengameart.org/run.py
View File

@ -8,6 +8,9 @@ import multiprocessing
h = HTML2Text() h = HTML2Text()
h.images_to_alt = True h.images_to_alt = True
counter = multiprocessing.Value("i", 0)
total = multiprocessing.Value("i", 0)
def request_timeout(url): def request_timeout(url):
while True: while True:
@ -41,18 +44,27 @@ def get_entries():
def download_entry(url): def download_entry(url):
r = request_timeout("https://opengameart.org" + url) global counter
soup = BeautifulSoup(r.text, "html.parser") global total
counter.value += 1
print(str(counter.value) + "/" + str(total.value) + " {0:.2f}".format(counter.value / total.value * 100) + "%")
simple_title = os.path.split(url)[1] simple_title = os.path.split(url)[1]
if not os.path.exists(simple_title + os.sep + "metadata.json"): if not os.path.exists("entries" + os.sep + simple_title + os.sep + "metadata.json"):
r = request_timeout("https://opengameart.org" + url)
soup = BeautifulSoup(r.text, "html.parser")
metadata = dict() metadata = dict()
try:
metadata["title"] = list(soup.find("div", attrs={"property": "dc:title"}).children)[0].text metadata["title"] = list(soup.find("div", attrs={"property": "dc:title"}).children)[0].text
metadata["tags"] = list() metadata["tags"] = list()
for tag in soup.find_all("a", attrs={"property": "rdfs:label skos:prefLabel"}): tag_list = soup.find_all("a", attrs={"property": "rdfs:label skos:prefLabel"})
if tag_list is not None:
for tag in tag_list:
metadata["tags"].append(tag.text) metadata["tags"].append(tag.text)
metadata["description"] = h.handle(str(soup.find("div", attrs={"class": "field-item even", "property": "content:encoded"}))).strip() metadata["description"] = h.handle(str(soup.find("div", attrs={"class": "field-item even", "property": "content:encoded"}))).strip()
@ -68,7 +80,6 @@ def download_entry(url):
link = file.find("a").get("href") link = file.find("a").get("href")
if not os.path.exists(path + os.sep + os.path.split(link)[1]): if not os.path.exists(path + os.sep + os.path.split(link)[1]):
print(link)
while True: while True:
try: try:
@ -84,18 +95,23 @@ def download_entry(url):
with open(path + os.sep + "metadata.json", "w") as f: with open(path + os.sep + "metadata.json", "w") as f:
json.dump(metadata, f) json.dump(metadata, f)
except:
print("ERROR " + url)
def download_all(): def download_all():
global total
pool = multiprocessing.Pool(processes=25) pool = multiprocessing.Pool(processes=25)
with open("entries.txt", "r") as f: with open("entries.txt", "r") as f:
pool.map(download_entry, f.read().splitlines()) lines = f.read().splitlines()
total.value = len(lines)
pool.map(download_entry, lines)
if not os.path.exists("entries"): if not os.path.exists("entries"):
os.mkdir("entries") os.mkdir("entries")
get_entries() # get_entries()
download_all() download_all()