diff --git a/config.toml b/config.toml index 7c262de..9db5c19 100644 --- a/config.toml +++ b/config.toml @@ -1,6 +1,6 @@ baseURL = "https://dataarchivist.net/" languageCode = "en-us" -title = "Data Archivist Weekly" +title = "Data Archivist Periodical" theme = "temple" pygmentsStyle = "native" diff --git a/content/posts/tt_1.md b/content/posts/tt_1.md new file mode 100644 index 0000000..83be885 --- /dev/null +++ b/content/posts/tt_1.md @@ -0,0 +1,76 @@ +--- +title: "Web scraping with task_tracker" +date: 2019-06-14T14:31:42-04:00 +draft: true +author: simon987 +--- + + + + +{{}} +import spotipy +from spotipy.oauth2 import SpotifyClientCredentials + +spotify = spotipy.Spotify(...) + + +def search_artist(name, mbid): + # Surround with quotes to get exact matches + name = '"' + name + '"' + + with open(os.devnull, 'w') as null: + # Silence spotipy's stdout + stdout = sys.stdout + sys.stdout = null + res = spotify.search(name, type="artist", limit=20) + sys.stdout = stdout + + with sqlite3.connect(dbfile) as conn: + conn.execute("INSERT INTO artist (mbid, query, data) VALUES (?,?,?)", (mbid, name, json.dumps(res))) + conn.commit() +{{}} + + +{{}} +try: + # This script is called like this: + # python ./run.py "" "" + task_str = sys.argv[1] + task = json.loads(task_str) + secret_str = sys.argv[2] + secret = json.loads(secret_str) + + CLIENT_ID = secret["CLIENT_ID"] + CLIENT_SECRET = secret["CLIENT_SECRET"] + client_credentials_manager = SpotifyClientCredentials(client_id=CLIENT_ID, client_secret=CLIENT_SECRET) + spotify = spotipy.Spotify(client_credentials_manager=client_credentials_manager) + + # This job's recipe is an array of name & mbid pairs + recipe = json.loads(task["recipe"]) + for job in recipe: + search_artist(job["name"], job["mbid"]) + +except Exception as e: + print(json.dumps({ + # Tell task_tracker that this task failed. It will be re-attempted later + "result": 1, + # Send full stack trace. It will be available in the Logs page. + "logs": [ + {"message": str(e) + traceback.format_exc(), "level": 3} + ] + })) + quit(2) + +print(json.dumps({ + "result": 0, +})) +{{}} + + + + +{{< figure src="/tt/new_project.png" title="New task_tracker project">}} +{{< figure src="/tt/secret.png" title="Project secret settings">}} +{{< figure src="/tt/hook.png" title="Gogs webhook configuration">}} +{{< figure src="/tt/perms.png" title="Private project require approval">}} diff --git a/static/tt/hook.png b/static/tt/hook.png new file mode 100644 index 0000000..30f7c2c Binary files /dev/null and b/static/tt/hook.png differ diff --git a/static/tt/new_project.png b/static/tt/new_project.png new file mode 100644 index 0000000..b3e3904 Binary files /dev/null and b/static/tt/new_project.png differ diff --git a/static/tt/perms.png b/static/tt/perms.png new file mode 100644 index 0000000..146a454 Binary files /dev/null and b/static/tt/perms.png differ diff --git a/static/tt/secret.png b/static/tt/secret.png new file mode 100644 index 0000000..e552172 Binary files /dev/null and b/static/tt/secret.png differ diff --git a/static/tt/spotify_data.png b/static/tt/spotify_data.png new file mode 100644 index 0000000..e642663 Binary files /dev/null and b/static/tt/spotify_data.png differ