commit 476491ca7b8b855cb60a4fbebe6dc86621363d2e Author: simon Date: Tue Nov 19 15:33:48 2019 -0500 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..249963d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.iml +.idea/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..0c5a0fa --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +Misc utility methods in Python \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db.py b/db.py new file mode 100644 index 0000000..e69de29 diff --git a/imhash.py b/imhash.py new file mode 100644 index 0000000..eadc530 --- /dev/null +++ b/imhash.py @@ -0,0 +1,8 @@ +import base64 + + +def b64hash(imhash, bcount): + """ImageHash hash to base64 string""" + return base64.b64encode( + sum(1 << i for i, b in enumerate(imhash.hash.flatten()) if b).to_bytes(bcount, "big") + ).decode("ascii") diff --git a/misc.py b/misc.py new file mode 100644 index 0000000..bfce03f --- /dev/null +++ b/misc.py @@ -0,0 +1,24 @@ +import time + +last_time_called = dict() + + +def rate_limit(per_second): + min_interval = 1.0 / float(per_second) + + def decorate(func): + last_time_called[func] = 0 + + def wrapper(*args, **kwargs): + elapsed = time.perf_counter() - last_time_called[func] + wait_time = min_interval - elapsed + if wait_time > 0: + time.sleep(wait_time) + + last_time_called[func] = time.perf_counter() + return func(*args, **kwargs) + + return wrapper + + return decorate + diff --git a/monitoring.py b/monitoring.py new file mode 100644 index 0000000..8db30e0 --- /dev/null +++ b/monitoring.py @@ -0,0 +1,30 @@ +import logging +import traceback + +from influxdb import InfluxDBClient + + +class Monitoring: + def __init__(self, db, host="localhost", logger=logging.getLogger("default")): + self._db = db + self._client = InfluxDBClient(host, 8086, "", "", db) + self._logger = logger + + self._init() + + def db_exists(self, name): + for db in self._client.get_list_database(): + if db["name"] == name: + return True + return False + + def _init(self): + if not self.db_exists(self._db): + self._client.create_database(self._db) + + def log(self, event): + try: + self._client.write_points(event) + except Exception as e: + self._logger.debug(traceback.format_exc()) + self._logger.error(str(e)) diff --git a/regex.py b/regex.py new file mode 100644 index 0000000..8c3268e --- /dev/null +++ b/regex.py @@ -0,0 +1,4 @@ +import re + +LINK_RE = re.compile(r"(https?://[\w\-_.]+\.[a-z]{2,4}([^\s<'\"]*|$))") +HTML_HREF_RE = re.compile(r"href=\"([^\"]+)\"") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..acc8ad3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ImageHash +influxdb \ No newline at end of file