Fixed file duplication problem

crawl_server/crawler.py

@@ -146,8 +146,8 @@ class RemoteDirectoryCrawler:
             try:
                 path = os.path.join(file.path, file.name, "")
                 if path not in self.crawled_paths:
-                    listing = directory.list_dir(path)
                     self.crawled_paths.add(path)
+                    listing = directory.list_dir(path)
 
                     for f in listing:
                         if f.is_dir:

debug_put.py

@@ -4,7 +4,7 @@ import json
 
 payload = json.dumps({
     "website_id": 123,
-    "url": "https://frenchy.ga/",
+    "url": "http://liminaire.fr/TEXTES/",
     "priority": 2,
     "callback_type": "",
     "callback_args": "{}"

search/search.py

@@ -82,7 +82,6 @@ class ElasticSearchEngine(SearchEngine):
     def import_json(self, in_str: str, website_id: int):
         import_every = 1000
 
-        print(in_str)
         docs = []
 
         for line in in_str.splitlines():