Initial commit

generate_scrape_tasks.py

@@ -0,0 +1,39 @@
+import csv
+import json
+from multiprocessing.pool import ThreadPool
+
+from task_tracker_drone.src.tt_drone.api import TaskTrackerApi, Worker
+
+TT_API_URL = "https://tt.simon987.net/api"
+TT_PROJECT = 1
+
+api = TaskTrackerApi(TT_API_URL)
+
+worker = Worker.from_file(api)
+if not worker:
+    worker = api.make_worker("last.fm scraper")
+    worker.dump_to_file()
+    worker.request_access(TT_PROJECT, True, True)
+
+    input("Give permission to " + worker.alias)
+
+with open("repo/artist.csv") as f:
+    reader = csv.reader(f)
+
+    def mktask(line):
+        res = worker.submit_task(
+            project=TT_PROJECT,
+            recipe=json.dumps({"mbid": line[0], "name": line[1]}),
+            unique_str=line[0],
+            max_assign_time=60 * 5,
+        )
+        print(res.text)
+
+    def lines():
+        for line in reader:
+            if "Group" in line[2]:
+                yield line
+
+    pool = ThreadPool(processes=60)
+    pool.map(func=mktask, iterable=lines())
+