diff --git a/od_db_client.py b/od_db_client.py index f3d5e00..13ebdb1 100644 --- a/od_db_client.py +++ b/od_db_client.py @@ -113,12 +113,14 @@ class OdDatabase: hl_path = truncate_path(hl_path, 65) hl_path += "/" if hl_path else "" + file_link = src["website_url"] + src["path"] + "/" + src["name"] + ("." if src["ext"] else "") + src["ext"] + message += "[" + src["website_url"] + "](https://od-db.the-eye.eu/website/" + str(src["website_id"]) + "/)" + hl_path - message += hl_name + ("." if src["ext"] else "") + src["ext"] + "| " + message += hl_name + ("." if src["ext"] else "") + src["ext"] + " [[DL]](" + file_link + ") | " message += humanfriendly.format_size(src["size"]) + " | " message += time.strftime("%Y-%m-%d", time.gmtime(src["mtime"])) + " \n" - message += "\n[More results for this query](https://the-eye.eu/search?q=" + query + ") |" \ + message += "\n[More results for this query](https://od-db.the-eye.eu/search?q=" + query + ") |" \ " [OD-Database](https://od-db.the-eye.eu/)" return message diff --git a/run.py b/run.py index 6627ed4..1abe52c 100644 --- a/run.py +++ b/run.py @@ -50,6 +50,7 @@ def process_url(comment, bot, url): if not is_valid_url(url): print("Url is invalid") handle_invalid_url(comment, bot, url) + return if od_db_client.website_is_blacklisted(url): print("Website is blacklisted") @@ -121,13 +122,48 @@ def handle_existing_website(comment, bot, website_id): bot.reply(comment, message) +def process_post(submission): + print("Checking new post with url " + submission.url) + url = os.path.join(submission.url, "") + + if not is_valid_url(url): + print("Url is invalid") + return + + if od_db_client.website_is_blacklisted(url): + print("Website is blacklisted") + return + + url = get_top_directory(url) + website_id = od_db_client.website_by_url(url) + + if not website_id: + print("Website does not exist") + + if not is_od(url): + print("Website is not an od") + return + + handle_new_website(submission, bot, url) + else: + print("Website already exists") + handle_existing_website(submission, bot, website_id) + + if __name__ == "__main__": reddit = praw.Reddit('opendirectories-bot', user_agent='github.com/simon987/opendirectories-bot-new (by /u/Hexahedr_n)') bot = RedditBot("processed.txt", reddit) subreddit = reddit.subreddit("test") + # Check comments for comment in subreddit.comments(limit=50): if not bot.has_crawled(comment): process_comment(comment, bot) + # Check submissions + for submission in subreddit.new(limit=3): + if not submission.is_self and not bot.has_crawled(submission): + process_post(submission) + +