From 19636abafede55c21481ece50b23bcc74f73bdc0 Mon Sep 17 00:00:00 2001
From: simon987 <me@simon987.net>
Date: Fri, 25 Dec 2020 20:12:15 -0500
Subject: [PATCH] stability tweaks

---
 docker-compose.yml | 1 +
 poal.py            | 5 +++++
 util.py            | 7 +++++--
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index afc75fb..60a0aa3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,3 +7,4 @@ services:
     environment:
       - "PF_REDIS_HOST="
       - "PF_RPS=1"
+      - "PF_MAX_PAGE=300"
diff --git a/poal.py b/poal.py
index 82e1dc1..68d5a01 100644
--- a/poal.py
+++ b/poal.py
@@ -1,4 +1,5 @@
 import json
+import os
 from json import JSONDecodeError
 from urllib.parse import urljoin
 
@@ -6,6 +7,8 @@ from post_process import get_links_from_body
 from state import PoalState
 from util import Web, logger
 
+PF_MAX_PAGE = os.environ.get("PF_MAX_PAGE", 9999999)
+
 
 class PoalHelper:
 
@@ -74,6 +77,8 @@ class PoalHelper:
                 page = 1
             else:
                 page = int(r.url.split("/")[-1])
+            if page + 1 > PF_MAX_PAGE:
+                return posts, None
             return posts, self.posts_url(board, page=page + 1)
         return posts, None
 
diff --git a/util.py b/util.py
index addea9d..e6d8d70 100644
--- a/util.py
+++ b/util.py
@@ -28,12 +28,15 @@ class Web:
                 kwargs["headers"]["User-Agent"] = UA
             else:
                 kwargs["headers"] = {"User-Agent": UA}
-            retries = 3
+            retries = 8
 
             while retries > 0:
                 retries -= 1
                 try:
-                    return self.session.get(url, **kwargs)
+                    r =  self.session.get(url, **kwargs)
+                    if r.status_code == 500:
+                        raise Exception("Server error")
+                    return r
                 except KeyboardInterrupt as e:
                     raise e
                 except Exception as e: