Crawl tasks are now fetched by the crawlers instead of pushed by the server

This commit is contained in:
Simon
2018-07-14 17:31:18 -04:00
parent d9e9f53f92
commit fe1d29aaea
20 changed files with 376 additions and 749 deletions

View File

@@ -91,8 +91,8 @@ def make_wide_filesystem(count=100000):
os.mkdir(new_path)
# dump_local_filesystem("/mnt/")
index_file_list("local_filesystem.json", 4)
dump_local_filesystem("/mnt/")
# index_file_list("local_filesystem.json", 4)
# random_searches(100000)
# dump_random_files(20000 * 100000)
# make_wide_filesystem(10000)