simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-10-31 13:46:52 +00:00
Code Issues Projects Releases Wiki Activity

Should fix some odd http listings

Browse Source
This commit is contained in:
Simon 2018-06-20 13:34:41 -04:00
parent cf51bb381c
commit c5deafbea5
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawl_server/remote_http.py
View File

@ -180,7 +180,8 @@ class HttpDirectory(RemoteDirectory):
@staticmethod @staticmethod
def _should_ignore(base_url, link: Anchor): def _should_ignore(base_url, link: Anchor):
if link.text == "../" or link.href == "../" or link.href.endswith(HttpDirectory.BLACK_LIST): if link.text == "../" or link.href == "../" or link.href == "./" \
or link.href.endswith(HttpDirectory.BLACK_LIST):
return True return True
# Ignore external links # Ignore external links