Added filter to check if a website can be scanned from its parent directory

crawl_server/remote_http.py

@@ -105,6 +105,7 @@ class HttpDirectory(RemoteDirectory):
         path_url = urljoin(self.base_url, path, "")
         body = self._stream_body(path_url)
         if not body:
+            logger.info("No body returned @ " + path_url)
             return None, None
         anchors = self._parse_links(body)