Add retry_sleep to retry

hexlib/env.py

@@ -20,7 +20,7 @@ def get_redis():
 def redis_publish(rdb, item, item_project,  item_type, item_subproject=None, item_category="x"):
 
     item_project = item_project.replace(".", "-")
-    item_subproject = item_subproject.replace(".", "-")
+    item_subproject = item_subproject.replace(".", "-") if item_subproject else None
 
     item_source = item_project if not item_subproject else f"{item_project}.{item_subproject}"
 
@@ -35,14 +35,16 @@ def redis_publish(rdb, item, item_project,  item_type, item_subproject=None, ite
         rdb.lpush(routing_key, item)
 
 
-def get_web():
+def get_web(session=None):
     ua = UserAgent()
 
     return Web(
+        session=session,
         proxy=os.environ.get("PROXY", None),
         rps=os.environ.get("RPS", 1),
         logger=stdout_logger,
         cookie_file=os.environ.get("COOKIE_FILE", None),
         retry_codes=set(int(x) if x else None for x in os.environ.get("RETRY_CODES", "").split(",")),
+        retry_sleep=int(os.environ.get("RETRY_SLEEP", 0)),
         ua=ua[os.environ.get("USER_AGENT")] if os.environ.get("USER_AGENT", None) is not None else None
     )

hexlib/misc.py

@@ -4,12 +4,14 @@ import time
 from threading import Lock
 
 import atexit
+from time import sleep
+
 import siphash
 
 last_time_called = dict()
 
 
-def retry(attempts, callback=None):
+def retry(attempts, callback=None, retry_sleep=0):
     def decorate(func):
         def wrapper(*args, **kwargs):
             retries = attempts
@@ -20,6 +22,7 @@ def retry(attempts, callback=None):
                     if callback:
                         callback(e)
                     retries -= 1
+                    sleep(retry_sleep)
         return wrapper
     return decorate
 

hexlib/web.py

@@ -112,7 +112,7 @@ def download_file(url, destination, session=None, headers=None, overwrite=False,
 
 
 class Web:
-    def __init__(self, proxy=None, rps=1, retries=3, logger=None, cookie_file=None, retry_codes=None, session=None,
+    def __init__(self, proxy=None, rps=1, retries=3, retry_sleep=0, logger=None, cookie_file=None, retry_codes=None, session=None,
                  ua=None):
         self._cookie_file = cookie_file
         self._proxy = proxy
@@ -140,7 +140,7 @@ class Web:
             }
 
         @rate_limit(rps)
-        @retry(retries, callback=self._error_callback)
+        @retry(retries, callback=self._error_callback, retry_sleep=retry_sleep)
         def get(url, **kwargs):
             self._current_req = "GET", url, kwargs
             r = self._session.get(url, **kwargs)

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="hexlib",
-    version="1.34",
+    version="1.37",
     description="Misc utility methods",
     author="simon987",
     author_email="me@simon987.net",