Update preprocess: now returns generator objects

This commit is contained in:
2021-09-19 09:35:35 -04:00
parent 4711cd1b66
commit d19442b00e
5 changed files with 65 additions and 78 deletions

10
bench/text.py Normal file
View File

@@ -0,0 +1,10 @@
from timeit import timeit
if __name__ == '__main__':
res = timeit(
setup="from hexlib.text import preprocess",
stmt='text = "x A b c d e f g h"\ncleaned = preprocess(\n text,\n lowercase=True,\n trigrams={\n ("a", "b", "c"),\n ("e", "f", "g"),\n }\n)'
)
print(res)