simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 02:46:45 +00:00
Code Issues Projects Releases Wiki Activity

Ignore 'parent directory' links

Browse Source
This commit is contained in:
Simon 2018-06-19 10:36:09 -04:00
parent 8f311e52ee
commit 8486555426
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/remote_http.py
View File

@ -177,7 +177,7 @@ class HttpDirectory(RemoteDirectory):
@staticmethod
def _should_ignore(base_url, link: Anchor):
if link.text == "../" or link.href.endswith(HttpDirectory.BLACK_LIST):
if link.text == "../" or link.href == "../" or link.href.endswith(HttpDirectory.BLACK_LIST):
return True
# Ignore external links