diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000..0cde934 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +tmux new -d -s mgCover bash -c "./task_runner.sh task_get_cover.py" + +tmux new -d -s mgLastfm bash -c "./task_runner.sh task_get_lastfm" +tmux new -d -s mgLastfm2 bash -c "./task_runner.sh task_get_lastfm" + +tmux new -d -s mgSpotify bash -c "./task_runner.sh task_get_spotify.py" +tmux new -d -s mgSpotify2 bash -c "./task_runner.sh task_get_spotify.py" diff --git a/mb_scratch.sql b/mb_scratch.sql index d07b2ce..0927353 100644 --- a/mb_scratch.sql +++ b/mb_scratch.sql @@ -125,25 +125,48 @@ VALUES ('label rename', 'WAS_RENAMED_TO'), ('label reissue', 'DOES_REISSUING_FOR'); +DROP VIEW mg.artist; CREATE OR REPLACE VIEW mg.artist AS SELECT gid as "id:ID(Artist)", - name, - fn_sortname(name, sort_name) as sortname, + artist.name, + fn_sortname(artist.name, sort_name) as sortname, COALESCE(begin_date_year, 0) as "year:int", comment, + COALESCE(lfm.listeners, 0) as "listeners", + COALESCE(lfm.playcount, 0) as "playcount", (CASE WHEN type = 2 THEN 'Group' ELSE 'Artist' END) as ":LABEL" -FROM artist; +FROM artist + LEFT JOIN mg.lastfm_artist lfa ON lfa.mbid = artist.gid + LEFT JOIN mg.lastfm_artist_meta lfm ON lfa.name = lfm.name; CREATE OR REPLACE VIEW mg.artist_artist AS SELECT a0.gid as ":START_ID(Artist)", a1.gid as ":END_ID(Artist)", + 0 as "weight:float", t.mg_name as ":TYPE" FROM l_artist_artist INNER JOIN artist a0 ON entity0 = a0.id INNER JOIN artist a1 ON entity1 = a1.id INNER JOIN link l on l.id = l_artist_artist.link INNER JOIN link_type lt ON lt.id = l.link_type - INNER JOIN mg.translate_artist_artist_rel t ON t.mb_name = lt.name; + INNER JOIN mg.translate_artist_artist_rel t ON t.mb_name = lt.name +UNION ALL +SELECT lfa0.mbid, + lfa1.mbid, + weight, + 'IS_RELATED_TO' +FROM mg.lastfm_artist_artist + INNER JOIN mg.lastfm_artist lfa0 ON lfa0.name = mg.lastfm_artist_artist.name0 + INNER JOIN mg.lastfm_artist lfa1 ON lfa1.name = mg.lastfm_artist_artist.name1 +UNION ALL +SELECT s0.mbid, + s1.mbid, + index::float, + 'IS_RELATED_TO' +FROM mg.spotify_artist_artist + INNER JOIN mg.spotify_artist s0 ON s0.spotid = mg.spotify_artist_artist.spotid0 + INNER JOIN mg.spotify_artist s1 ON s1.spotid = mg.spotify_artist_artist.spotid1; + CREATE OR REPLACE VIEW mg.release AS @@ -189,21 +212,27 @@ CREATE OR REPLACE VIEW mg.tag AS WITH occurences AS ( SELECT tag, COUNT(*) as count FROM ( - SELECT tag + SELECT name as tag FROM release_group_tag + INNER JOIN tag t ON t.id = tag + UNION ALL + SELECT name + FROM release_tag + INNER JOIN tag t ON t.id = tag + UNION ALL + SELECT lower(name) + FROM mg.lastfm_artist_tag UNION ALL SELECT tag - FROM release_tag + FROM mg.spotify_artist_tag ) as tags GROUP BY tag ) -SELECT tag.id as "id:ID(Tag)", - tag.name, - occurences.count as "occurences:int" -FROM tag - INNER JOIN occurences ON occurences.tag = tag.id -WHERE ref_count > 0 - AND occurences.count > 5; +SELECT row_number() over (ORDER BY tag) as "id:ID(Tag)", + tag, + count as "occurences:int" +FROM occurences +WHERE count > 5; CREATE OR REPLACE VIEW mg.release_tag AS @@ -230,7 +259,7 @@ SELECT a.gid as ":START_ID(Artist greatest(least(artist_tag.count::float / 8, 1), 0.2) as "weight:float" FROM artist_tag INNER JOIN artist a on artist_tag.artist = a.id - INNER JOIN mg.tag t ON t."id:ID(Tag)" = artist_tag.tag + INNER JOIN mg.tag t ON t."id:ID(Tag)" = artist_tag.tag; CREATE OR REPLACE VIEW mg.tag_tag AS SELECT tag_relation.tag1 as ":START_ID(Tag)", @@ -271,8 +300,7 @@ FROM l_label_label INNER JOIN label l1 on l_label_label.entity1 = l1.id INNER JOIN link l on l.id = l_label_label.link INNER JOIN link_type lt ON lt.id = l.link_type - INNER JOIN mg.translate_label_label_rel t ON t.mb_name = lt.name - + INNER JOIN mg.translate_label_label_rel t ON t.mb_name = lt.name; -------------- @@ -383,9 +411,11 @@ CREATE TABLE mg.spotify_raw_data CREATE OR REPLACE FUNCTION asciifold(text) RETURNS text AS '/pglib/libasciifolding.so', -'asciifold' LANGUAGE C STRICT; +'asciifold' LANGUAGE C STRICT + PARALLEL SAFE; CREATE OR REPLACE FUNCTION asciifold_lower(text) RETURNS text AS '/pglib/libasciifolding.so', -'asciifold_lower' LANGUAGE C STRICT; +'asciifold_lower' LANGUAGE C STRICT + PARALLEL SAFE; diff --git a/task_get_lastfm.py b/task_get_lastfm.py index 328e552..c756a51 100755 --- a/task_get_lastfm.py +++ b/task_get_lastfm.py @@ -31,7 +31,7 @@ def save_tags(lfm_name, tags): cur.execute("DELETE FROM mg.lastfm_artist_tag WHERE name=%s", (lfm_name,)) cur.execute( "INSERT INTO mg.lastfm_artist_tag VALUES %s" % - ",".join("('%s', '%s')" % (n, t) for (n, t) in zip(repeat(lfm_name), tags)) + ",".join("('%s', '%s')" % (n, t.strip()) for (n, t) in zip(repeat(lfm_name), tags)) ) diff --git a/task_runner.sh b/task_runner.sh new file mode 100755 index 0000000..4717a3c --- /dev/null +++ b/task_runner.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +if [[ -z "${LASTFM_APIKEY}" ]]; then + echo "LASTFM_APIKEY is not set" + exit +fi + +if [[ -z "${SPOTIFY_CLIENTID}" ]]; then + echo "SPOTIFY_CLIENTID is not set" + exit +fi + +if [[ -z "${SPOTIFY_SECRET}" ]]; then + echo "SPOTIFY_SECRET is not set" + exit +fi + +while true; do + /usr/bin/time python "$1" --count 50 &>> "$1".log +done