stability tweaks

docker-compose.yml

@@ -7,3 +7,4 @@ services:
     environment:
       - "PF_REDIS_HOST="
       - "PF_RPS=1"
+      - "PF_MAX_PAGE=300"

poal.py

@@ -1,4 +1,5 @@
 import json
+import os
 from json import JSONDecodeError
 from urllib.parse import urljoin
 
@@ -6,6 +7,8 @@ from post_process import get_links_from_body
 from state import PoalState
 from util import Web, logger
 
+PF_MAX_PAGE = os.environ.get("PF_MAX_PAGE", 9999999)
+
 
 class PoalHelper:
 
@@ -74,6 +77,8 @@ class PoalHelper:
                 page = 1
             else:
                 page = int(r.url.split("/")[-1])
+            if page + 1 > PF_MAX_PAGE:
+                return posts, None
             return posts, self.posts_url(board, page=page + 1)
         return posts, None
 

util.py

@@ -28,12 +28,15 @@ class Web:
                 kwargs["headers"]["User-Agent"] = UA
             else:
                 kwargs["headers"] = {"User-Agent": UA}
-            retries = 3
+            retries = 8
 
             while retries > 0:
                 retries -= 1
                 try:
-                    return self.session.get(url, **kwargs)
+                    r =  self.session.get(url, **kwargs)
+                    if r.status_code == 500:
+                        raise Exception("Server error")
+                    return r
                 except KeyboardInterrupt as e:
                     raise e
                 except Exception as e: