8kun hotfix

chan/chan.py

@@ -384,68 +384,39 @@ CHANS = {
         ),
         rps=1 / 60
     ),
-    "8kun": JsonKunChanHelper(
+    "8kun2": JsonKunChanHelper(
-        29,
+        35,
-        "https://8kun.net/",
+        "https://8kun.top/",
-        "https://media.8kun.net/",
+        "https://media.8kun.top/",
         "/res/",
         "file_store/",
         (
-            "1",
+            "1", "55chan", "64chen", "8bantb", "8tube", "a", "abdl2", "agdg", "amv", "aneki", "animu", "animus",
-            "a",
+            "ara", "arda", "arms", "asatru", "asmr", "aus", "ausneets", "b", "baka", "baneposting", "baseballbat",
-            "aneki",
+            "bcards", "bleached", "blog", "bonehurtingjuice", "bq", "brit", "bubblegum", "builders", "bunkers", "butt",
-            "ara",
+            "cafechan", "caffe", "canada", "cath", "chori", "choroy", "christian", "christianity", "christianmeme",
-            "asatru",
+            "cicachan", "civicrs", "ck", "cloveros", "co", "cow", "cuckquean", "cute", "cyber", "cyoa", "czech",
-            "asmr",
+            "dadtalk", "danpu", "dao101", "degen", "delete", "dempart", "desu", "diaperfags", "diaperfetish", "dir",
-            "aus",
+            "dolphin", "dpfag", "dpr", "druid", "e9y", "eatme", "ebola", "eerie", "egy", "egypt", "etika", "eu",
-            "b",
+            "euskotxa", "exit", "f1", "fa", "fairy", "fallen", "fast", "faygo", "feet", "femaledomination", "feri",
-            "baka",
+            "fightcomms", "film", "flemish", "floss", "fortnite", "freedomzine", "fukemo", "fumo", "fur", "furry", "g",
-            "brit",
+            "gamergatehq", "genesis", "gesu", "ggis", "girltalk", "greenbreeze", "gts", "haxxor", "hentai",
-            "builders",
+            "hentaiclub", "herm", "hermetics", "hgb", "hgg", "hindu", "hisparefugio", "hissss", "hnt", "hover",
-            "cafechan",
+            "hybrids", "hydrus", "hypno", "hypnochan", "icup", "imperium", "in", "ipfs", "ircsecrets", "islam", "ita",
-            "canada",
+            "jaooo", "jewess", "jmaatv", "joker", "jp", "k", "kekforceusa", "kemono", "kocsog", "kohlchan", "komica",
-            "choroy",
+            "komika", "kpop", "lain", "lego", "leo", "lewd", "lit", "lol", "loomis", "loroy", "luddite", "magick",
-            "co",
+            "maka", "mde", "merrychristmas", "miku", "milf", "mom", "monster", "msb", "mtb", "mtt", "mu", "n0thingness",
-            "cuckquean",
+            "nanachi", "natiofr", "nep", "newbrit", "newsplus", "nobody", "nofap", "nofur", "nogatco", "nothingness",
-            "cute",
+            "ntr", "nuke8", "oanda", "ocb", "ocult", "omorashi", "opmk", "os", "otter", "p", "panconleche", "pdfs",
-            "cyoa",
+            "peaceofmind", "pen", "philosophy", "pkmns", "pnd", "pokeporn", "polymath", "pone", "projectdcomms",
-            "dempart",
+            "pyatibrat", "qm", "qpatriotresearch", "qresearch", "qrnews", "rand21", "rec", "rmart", "rusrandom",
-            "e9y",
+            "rzabczan", "s", "s8s", "sag", "sapphic", "shousa", "sikhi", "sip", "sl", "snowboarding", "socpl", "strek",
-            "egy", "hgg",
+            "subs", "sve", "t", "tan", "tdt", "tech9", "techan", "techbunker", "tek", "templeos", "tenda", "teraha",
-            "f1",
+            "texit", "tf2", "tg", "thb", "thedickshow", "throat", "tibby", "tikilounge", "tkr", "tr55",
-            "fa",
+            "trashcollector", "truthlegion", "tulpamancers", "turul", "tutturu", "tv", "u", "uaco", "ucla",
-            "feet",
+            "underground", "usersunion", "v", "vichan", "vietkong", "vietnam", "vore", "vr", "warposting", "wdsc",
-            "film",
+            "webm", "wg", "wga", "wikieat", "wis", "wmafsex", "workrelated", "wqt", "wx", "x", "xivl", "xtian",
-            "furry",
+            "zoomerright", "zundel",
-            "hentai",
-            "hisparefugio",
-            "hypno",
-            "islam",
-            "ita",
-            "jp",
-            "komika",
-            "lewd",
-            "lit",
-            "magick",
-            "mde",
-            "monster",
-            "mu",
-            "newbrit",
-            "newsplus",
-            "ntr",
-            "philosophy",
-            "qresearch",
-            "rzabczan",
-            "s",
-            "sl",
-            "tg",
-            "turul",
-            "tv",
-            "v",
-            "vichan",
-            "vr",
-            "wx",
-            "x",
         ),
         rps=1
     ),
@@ -625,4 +596,5 @@ CHANS = {
         ),
         rps=1 / 3
     ),
+    # next is 36
 }

chan/helper.py

@@ -10,6 +10,7 @@ class ChanHelper:
         self._image_path = image_path
         self._boards = boards
         self.rps = rps
+        self.get_method = None
 
     def boards(self):
         return [b.replace("\\_", "_") for b in self._boards if not b.startswith("_")]

chan/json_kun.py

@@ -1,8 +1,29 @@
+from vanwanet_scrape.scraper import Scraper
+
 from chan.json import JsonChanHelper
+from util import logger
 
 
 class JsonKunChanHelper(JsonChanHelper):
 
+    def __init__(self, db_id, base_url, image_url, thread_path, image_path, boards, rps):
+        super().__init__(db_id, base_url, image_url, thread_path, image_path, boards, rps)
+
+        self._scraper = Scraper(
+            headers={
+                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/70.0",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Referer": "https://8kun.top/index.html"
+            },
+            domains=[
+                "8kun.top",
+                "media.8kun.top",
+                "sys.8kun.net"
+            ],
+            logger=logger
+        )
+
+        self.get_method = self._scraper.get
+
     def image_url(self, board, tim, extension):
         return "%s%s%s%s" % (self._image_url, self._image_path, tim, extension)
-

run.py

@@ -21,7 +21,7 @@ BYPASS_RPS = False
 
 class ChanScanner:
     def __init__(self, helper, proxy):
-        self.web = Web(monitoring if MONITORING else None, rps=helper.rps, proxy=proxy)
+        self.web = Web(monitoring if MONITORING else None, rps=helper.rps, get_method=helper.get_method, proxy=proxy)
         self.helper = helper
         self.state = ChanState()
 
@@ -142,7 +142,7 @@ class ChanState:
 
 def publish_worker(queue: Queue, helper, p):
     channel = connect()
-    web = Web(monitoring if MONITORING else None, rps=helper.rps, proxy=p)
+    web = Web(monitoring if MONITORING else None, rps=helper.rps, get_method=helper.get_method, proxy=p)
 
     while True:
         try:

util.py

@@ -25,13 +25,14 @@ logger.addHandler(StreamHandler(sys.stdout))
 
 
 class Web:
-    def __init__(self, monitoring, rps=1 / 2, proxy=None):
+    def __init__(self, monitoring, rps=1 / 2, proxy=None, get_method=None):
         self.session = requests.Session()
         if proxy:
             self.session.proxies = {"http": proxy, "https": proxy}
             self.session.verify = False
         self._rps = rps
         self.monitoring = monitoring
+        self._get_method = get_method
 
         @rate_limit(self._rps)
         def _get(url, **kwargs):
@@ -40,6 +41,8 @@ class Web:
             while retries > 0:
                 retries -= 1
                 try:
+                    if self._get_method:
+                        return self._get_method(url, **kwargs)
                     return self.session.get(url, **kwargs)
                 except Exception as e:
                     logger.warning("Error with request %s: %s" % (url, str(e)))