misc fixes

chan.py

@@ -1,3 +1,4 @@
+import datetime
 import json
 from urllib.parse import urljoin
 
@@ -13,7 +14,10 @@ class ChanHelper:
         self._image_url = image_url
         self._thread_path = thread_path
         self._image_path = image_path
-        self.boards = boards
+        self._boards = boards
+
+    def boards(self):
+        return [b for b in self._boards if not b.startswith("_")]
 
     def image_url(self, board, tim, extension):
         return "%s%s%s%s%s" % (self._image_url, board, self._image_path, tim, extension)
@@ -25,7 +29,7 @@ class ChanHelper:
         return "%s%s%s%d.json" % (self._base_url, board, self._thread_path, thread)
 
     def board_hash(self, board):
-        return str((self.boards.index(board) + 1) * 10000)
+        return str((self._boards.index(board) + 1) * 10000)
 
     @staticmethod
     def item_id(item):
@@ -82,7 +86,11 @@ class HtmlChanHelper(ChanHelper):
         return -1
 
     @staticmethod
-    def item_id(item):
+    def item_mtime(item):
+        if item is None:
+            return int(datetime.datetime.now().timestamp())
+        print(item)
+        exit(0)
         return 0  # TODO
 
     def parse_threads_list(self, r):
@@ -126,7 +134,7 @@ class JsonChanHelper(ChanHelper):
         return item["no"]
 
     @staticmethod
-    def item_id(item):
+    def item_mtime(item):
         return item["time"]
 
     def item_urls(self, item, board):
@@ -234,8 +242,8 @@ CHANS = {
         "/src/",
         (
             "λ", "diy", "sec", "tech", "inter", "lit", "music", "vis",
-            "hum", "drg", "zzz", "layer", "q", "r", "cult", "psy",
+            "hum", "drg", "zzz", "layer", "q", "r", "_cult", "_psy",
-            "mega",
+            "_mega",
         )
     ),
     "uboachan": JsonChanHelper(

monitoring.py

@@ -1,5 +1,9 @@
+import traceback
+
 from influxdb import InfluxDBClient
 
+from util import logger
+
 client = InfluxDBClient("localhost", 8086, "root", "root", "chan_feed")
 
 
@@ -15,4 +19,8 @@ def init():
 
 
 def log(event):
-    client.write_points(event)
+    try:
+        client.write_points(event)
+    except Exception as e:
+        logger.debug(traceback.format_exc())
+        logger.error(str(e))

run.py

@@ -52,7 +52,7 @@ class ChanScanner:
                 self.state.mark_thread_as_visited(thread, self.helper, board)
 
     def all_posts(self):
-        for board in self.helper.boards:
+        for board in self.helper.boards():
             for post in self._posts(board):
                 yield post, board
 

util.py

@@ -55,6 +55,9 @@ class Web:
                 "time": str(datetime.utcnow()),
                 "fields": {
                     "status_code": r.status_code
-                }
+                },
+                "tags": {
+                    "ok": r.status_code == 200
+                },
             }])
         return r