simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 18:36:44 +00:00
Code Issues Projects Releases Wiki Activity

Fixed file duplication problem

Browse Source
This commit is contained in:
Simon 2018-06-12 15:55:52 -04:00
parent 1718bb91ca
commit af2601ee70
3 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/crawler.py
View File

@ -146,8 +146,8 @@ class RemoteDirectoryCrawler:
try: try:
path = os.path.join(file.path, file.name, "") path = os.path.join(file.path, file.name, "")
if path not in self.crawled_paths: if path not in self.crawled_paths:
listing = directory.list_dir(path)
self.crawled_paths.add(path) self.crawled_paths.add(path)
listing = directory.list_dir(path)
for f in listing: for f in listing:
if f.is_dir: if f.is_dir:

2
debug_put.py
View File

@ -4,7 +4,7 @@ import json
payload = json.dumps({ payload = json.dumps({
"website_id": 123, "website_id": 123,
"url": "https://frenchy.ga/", "url": "http://liminaire.fr/TEXTES/",
"priority": 2, "priority": 2,
"callback_type": "", "callback_type": "",
"callback_args": "{}" "callback_args": "{}"

1
search/search.py
View File

@ -82,7 +82,6 @@ class ElasticSearchEngine(SearchEngine):
def import_json(self, in_str: str, website_id: int): def import_json(self, in_str: str, website_id: int):
import_every = 1000 import_every = 1000
print(in_str)
docs = [] docs = []
for line in in_str.splitlines(): for line in in_str.splitlines():