add pgsql cursor util & silent stdout

hexlib/db.py

@@ -129,3 +129,20 @@ def _deserialize(value, col_type):
     if col_type == "blob":
         return base64.b64decode(value)
     return value
+
+
+def pg_fetch_cursor_all(cur, name, batch_size=1000):
+
+    while True:
+        cur.execute("FETCH FORWARD %s FROM %s", (batch_size, name))
+        cnt = 0
+
+        for row in cur:
+            cnt += 1
+            yield row
+
+        if cnt != batch_size:
+            cur.execute("FETCH ALL FROM %s", (batch_size, name))
+            for row in cur:
+                yield row
+            break

hexlib/misc.py

@@ -1,4 +1,6 @@
 import time
+import os
+import sys
 
 import atexit
 import siphash
@@ -28,7 +30,6 @@ def rate_limit(per_second):
 
 
 def buffered(batch_size: int, flush_on_exit: bool = False):
-
     def decorate(func):
         buffer = []
         lock = Lock()
@@ -58,3 +59,12 @@ def strhash(str):
 
 def signed64(i):
     return -(i & 0x8000000000000000) | (i & 0x7fffffffffffffff)
+
+
+def silent_stdout(func, *args, **kwargs):
+    with open(os.devnull, 'w') as null:
+        stdout = sys.stdout
+        sys.stdout = null
+        res = func(*args, **kwargs)
+        sys.stdout = stdout
+    return res

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="hexlib",
-    version="1.8",
+    version="1.9",
     description="Misc utility methods",
     author="simon987",
     author_email="me@simon987.net",