simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 23:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Removed unsuitable scrapy spider and implemented custom crawler

Browse Source
This commit is contained in:
Simon
2018-06-10 20:08:59 -04:00
parent d8c16d53e6
commit f2d914060b
6 changed files with 129 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app.py
View File

@@ -157,8 +157,7 @@ def try_enqueue(url):
return "A parent directory of this url has already been posted", "danger"
if not od_util.is_valid_url(url):
return "<strong>Error:</strong> Invalid url. Make sure to include the http(s):// suffix. " \
"FTP is not supported", "danger"
return "<strong>Error:</strong> Invalid url. Make sure to include the appropriate scheme.", "danger"
if od_util.is_blacklisted(url):