simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 02:16:47 +00:00
Code Issues Projects Releases Wiki Activity

Small bugfix

Browse Source
This commit is contained in:
Simon 2018-08-23 12:12:23 -04:00
parent 54b4d2d5b4
commit cadaf14c1b
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
crawl_server/remote_http.py
View File

@ -119,7 +119,6 @@ class HttpDirectory(RemoteDirectory):
return curl_head
def list_dir(self, path):
current_dir_name = path[path.rstrip("/").rfind("/") + 1: -1]
@ -235,14 +234,14 @@ class HttpDirectory(RemoteDirectory):
@staticmethod
def _should_ignore(base_url, current_path, link: Anchor):
if urljoin(base_url, link.href) == urljoin(urljoin(base_url, current_path), "../"):
full_url = urljoin(base_url, link.href)
if full_url == urljoin(urljoin(base_url, current_path), "../") or full_url == base_url:
return True
if link.href.endswith(HttpDirectory.BLACK_LIST):
return True
# Ignore external links
full_url = urljoin(base_url, link.href)
if not full_url.startswith(base_url):
return True