simon987/opendirectories-bot
1
0
Fork 0
mirror of https://github.com/simon987/opendirectories-bot.git synced 2025-04-19 10:16:47 +00:00
Code Issues Projects Releases Wiki Activity

Fixes problem with urls with no trailing slash

Browse Source
This commit is contained in:
simon987 2018-03-26 18:32:38 -04:00
parent 2d1261c0f1
commit 3596829b06
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawler.py
View File

@ -1,6 +1,7 @@
import requests import requests
from parser import NginxParser, ApacheParser from parser import NginxParser, ApacheParser
from crawl_report import ReportSaver, ReportBuilder from crawl_report import ReportSaver, ReportBuilder
import os
class Crawler: class Crawler:
@ -8,7 +9,7 @@ class Crawler:
def __init__(self, url, test_url): def __init__(self, url, test_url):
self.files = [] self.files = []
self.parsed_urls = [] self.parsed_urls = []
self.base_url = url self.base_url = os.path.join(url, '')
if url.startswith("http"): if url.startswith("http"):
if test_url: if test_url: