mirror of
https://github.com/simon987/music-graph-ui.git
synced 2025-11-04 05:06:51 +00:00
49 lines
1.2 KiB
Python
49 lines
1.2 KiB
Python
import csv
|
|
import json
|
|
from multiprocessing.pool import ThreadPool
|
|
|
|
from task_tracker_drone.src.tt_drone.api import TaskTrackerApi, Worker
|
|
|
|
TT_API_URL = "https://tt.simon987.net/api"
|
|
TT_PROJECT = 1
|
|
|
|
api = TaskTrackerApi(TT_API_URL)
|
|
|
|
worker = Worker.from_file(api)
|
|
if not worker:
|
|
worker = api.make_worker("last.fm scraper")
|
|
worker.dump_to_file()
|
|
worker.request_access(TT_PROJECT, True, True)
|
|
input("Give permission to " + worker.alias)
|
|
|
|
with open("repo/artist.csv") as f:
|
|
reader = csv.reader(f)
|
|
|
|
def mktask(lines):
|
|
res = worker.submit_task(
|
|
project=TT_PROJECT,
|
|
recipe=json.dumps(
|
|
[{"mbid": line[0], "name": line[1]} for line in lines]
|
|
),
|
|
unique_str=lines[0][0],
|
|
max_assign_time=60 * 5,
|
|
)
|
|
print(res.text)
|
|
|
|
def lines():
|
|
line_batch = list()
|
|
|
|
for line in reader:
|
|
if "Group" in line[3]:
|
|
line_batch.append(line)
|
|
if len(line_batch) >= 30:
|
|
res = list(line_batch)
|
|
line_batch.clear()
|
|
yield res
|
|
|
|
tasks = list(lines())
|
|
|
|
pool = ThreadPool(processes=25)
|
|
pool.map(func=mktask, iterable=tasks)
|
|
|