Added filter to check if a website can be scanned from its parent directory

crawl_server/crawler.py

@@ -92,6 +92,7 @@ class RemoteDirectoryCrawler:
                 if root_listing:
                     self.crawled_paths.append(path_id)
                 else:
+                    logger.info("No files in root listing for " + self.url)
                     return CrawlResult(0, "empty")
                 directory.close()
             except TimeoutError:

crawl_server/remote_http.py

@@ -105,6 +105,7 @@ class HttpDirectory(RemoteDirectory):
         path_url = urljoin(self.base_url, path, "")
         body = self._stream_body(path_url)
         if not body:
+            logger.info("No body returned @ " + path_url)
             return None, None
         anchors = self._parse_links(body)