Add two Lynx chans, update dependencies

chan/alokal_json.py

@@ -1,4 +1,4 @@
-from chan.json import JsonChanHelper
+from chan.chan_json import JsonChanHelper
 from post_process import get_links_from_body
 
 

chan/chan.py

@@ -9,10 +9,11 @@ from chan.endchan_html import EndchanHtmlChanHelper
 from chan.fchan_html import FChanHtmlChanHelper
 from chan.hispachan_html import HispachanHtmlHelper
 from chan.iichan_html import IichanHtmlChanHelper
-from chan.json import JsonChanHelper
+from chan.chan_json import JsonChanHelper
 from chan.json_kun import JsonKunChanHelper
 from chan.kev4_php import Kev4PhpHelper
 from chan.lolnada_html import LolNadaHtmlChanHelper
+from chan.lynx import LynxChanHelper
 from chan.mayuri import MayuriChanHelper
 from chan.nowere_html import NowereHtmlChanHelper
 from chan.plus4chan_html import Plus4ChanHelper
@@ -596,5 +597,26 @@ CHANS = {
         ),
         rps=1 / 3
     ),
-    # next is 36
+    "waifuist": LynxChanHelper(
+        36,
+        "https://waifuist.pro/",
+        "https://waifuist.pro/",
+        "/res/",
+        "",
+        (
+            "w", "starlet", "etc",
+        ),
+        rps=1 / 25
+    ),
+    "cutiegarden": LynxChanHelper(
+        37,
+        "https://cutie.garden/",
+        "https://cutie.garden/",
+        "/res/",
+        "",
+        (
+            "lg", "cozy", "meta", "test"
+        ),
+        rps=1 / 25
+    ),
 }

chan/json_kun.py

@@ -1,6 +1,6 @@
 from vanwanet_scrape.scraper import Scraper
 
-from chan.json import JsonChanHelper
+from chan.chan_json import JsonChanHelper
 from util import logger
 
 

chan/lynx.py

@@ -0,0 +1,83 @@
+import json
+from datetime import datetime
+from json import JSONDecodeError
+from urllib.parse import urljoin
+
+import cloudscraper
+
+from chan.helper import ChanHelper
+from util import logger
+
+
+class LynxChanHelper(ChanHelper):
+    """See https://gitgud.io/LynxChan/LynxChan/blob/master/doc/Json.txt"""
+
+    def __init__(self, db_id, base_url, image_url, thread_path, image_path, boards, rps):
+        super().__init__(db_id, base_url, image_url, thread_path, image_path, boards, rps)
+
+        scraper = cloudscraper.create_scraper()
+        self.get_method = scraper.get
+
+    @staticmethod
+    def item_id(item):
+        return item["threadId"] if LynxChanHelper.item_type(item) == "thread" else item["postId"]
+
+    @staticmethod
+    def item_mtime(item):
+        return datetime.fromisoformat(item["creation"]).timestamp()
+
+    def item_urls(self, item, board):
+        return [
+            urljoin(self._base_url, im["path"])
+            for im in item["files"]
+        ] if "files" in item and item["files"] else []
+
+    @staticmethod
+    def item_type(item):
+        return "thread" if "threadId" in item else "post"
+
+    def threads_url(self, board):
+        return "%s%s/1.json" % (self._base_url, board)
+
+    @staticmethod
+    def thread_mtime(thread):
+        return (thread["ommitedPosts"] if "ommitedPosts" in thread else 0) + len(thread["posts"])
+
+    @staticmethod
+    def parse_threads_list(r):
+        try:
+            j = json.loads(r.content.decode('utf-8', 'ignore'))
+            if len(j) == 0 or "threads" not in j:
+                logger.warning("No threads in response for %s: %s" % (r.url, r.text,))
+                return [], None
+        except JSONDecodeError:
+            logger.warning("JSONDecodeError for %s:" % (r.url,))
+            logger.warning(r.text)
+            return [], None
+
+        next_page = None
+        url = r.url[:r.url.rfind("?")] if "?" in r.url else r.url
+        current_page = int(url[url.rfind("/") + 1:-5])
+        if current_page < j["pageCount"]:
+            next_page = urljoin(r.url, "%d.json" % (current_page + 1))
+
+        return j["threads"], next_page
+
+    @staticmethod
+    def parse_thread(r):
+        try:
+            j = json.loads(r.content.decode('utf-8', 'ignore'))
+        except JSONDecodeError:
+            logger.warning("JSONDecodeError for %s:" % (r.url,))
+            logger.warning(r.text)
+            return []
+
+        all_items = []
+        for post in j["posts"]:
+            post["_parent"] = j["threadId"]
+            all_items.append(post)
+
+        del j["posts"]
+        all_items.append(j)
+
+        return all_items

chan/synch_json.py

@@ -1,4 +1,4 @@
-from chan.json import JsonChanHelper
+from chan.chan_json import JsonChanHelper
 from post_process import get_links_from_body