simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-11 14:08:51 +00:00
Code Issues Projects Releases Wiki Activity

Improved external link detection

Browse Source
This commit is contained in:
Simon
2018-06-18 12:14:05 -04:00
parent 400abc9a3c
commit b63c7190c3
2 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crawl_server/remote_http.py
View File

@@ -1,4 +1,4 @@
from urllib.parse import unquote
from urllib.parse import unquote, urljoin
import os
from html.parser import HTMLParser
from itertools import repeat
@@ -177,7 +177,8 @@ class HttpDirectory(RemoteDirectory):
return True
# Ignore external links
if link.href.startswith("http") and not link.href.startswith(base_url):
full_url = os.path.join(base_url, link.href)
if not full_url.startswith(base_url):
return True
def close(self):