diff --git a/README.md b/README.md index 12f2ac9..141a8e3 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ image boards and publishes serialised JSON to RabbitMQ for real-time ingest. Compatible image boards: 4chan, lainchan, uboachan, -22chan, wizchan, 1chan, 2ch.hk, endchan, 38chan. +22chan, wizchan, 1chan, 2ch.hk, endchan, 38chan, alokal. Can optionally push monitoring data to InfluxDB. Below is an example of Grafana being used to display it. diff --git a/chan.py b/chan.py index 944a3dd..a83c752 100644 --- a/chan.py +++ b/chan.py @@ -160,7 +160,7 @@ class JsonChanHelper(ChanHelper): try: j = json.loads(r.text) except JSONDecodeError: - logger.warning("JSONDecodeError for %s:" % (r.url, )) + logger.warning("JSONDecodeError for %s:" % (r.url,)) logger.warning(r.text) return [], None @@ -191,7 +191,7 @@ class RussianJsonChanHelper(ChanHelper): try: j = json.loads(r.text) except JSONDecodeError: - logger.warning("JSONDecodeError for %s:" % (r.url, )) + logger.warning("JSONDecodeError for %s:" % (r.url,)) logger.warning(r.text) return [], None return j["threads"], None @@ -225,6 +225,22 @@ class RussianJsonChanHelper(ChanHelper): return list(urls) +class AlokalJsonChanHelper(JsonChanHelper): + + def item_urls(self, item, board): + urls = set() + + if "com" in item and item["com"]: + urls.update(get_links_from_body(item["com"])) + elif "sub" in item and item["sub"]: + urls.update(get_links_from_body(item["sub"])) + if "fsize" in item and item["fsize"]: + urls.add(self._base_url + self._image_path + item["tim"] + "/" + str(item["no"]) + item["ext"]) + + return list(urls) + + + CHANS = { "4chan": JsonChanHelper( 1, @@ -242,7 +258,7 @@ CHANS = { "news", "out", "po", "pol", "qst", "sci", "soc", "sp", "tg", "toy", "trv", "tv", "vp", "wsg", "wsr", "x" ), - rps=3/2 + rps=3 / 2 ), "lainchan": JsonChanHelper( 2, @@ -355,5 +371,17 @@ CHANS = { "a", "b", "g", "38" ), rps=1 / 600 + ), + "alokal": AlokalJsonChanHelper( + 10, + "https://alokal.eu/", + "https://alokal.eu/", + "/", + "src/", + ( + "b", "pol", "sk", "int", "slav", "s", "gv", "mda", "sp", + "fit", "had", + ), + rps=1 / 4 ) } diff --git a/post_process.py b/post_process.py index 2ec8641..0707b31 100644 --- a/post_process.py +++ b/post_process.py @@ -40,7 +40,7 @@ def b64hash(imhash, bcount): def image_meta(url, url_idx, web): r = web.get(url) if not r: - logger.warning("") + logger.warning("Could not download image") return None buf = r.content