Added filter to check if a website can be scanned from its parent directory

2025-12-14 07:09:03 +00:00 · 2018-07-10 10:14:23 -04:00
parent f226b82f5a
commit d138db8f06
4 changed files with 46 additions and 9 deletions
--- a/crawl_server/crawler.py
+++ b/crawl_server/crawler.py
@@ -92,6 +92,7 @@ class RemoteDirectoryCrawler:
                if root_listing:
                    self.crawled_paths.append(path_id)
                else:
+                    logger.info("No files in root listing for " + self.url)
                    return CrawlResult(0, "empty")
                directory.close()
            except TimeoutError: