simon987/opendirectories-bot
1
0
Fork 0
mirror of https://github.com/simon987/opendirectories-bot.git synced 2025-12-14 15:49:03 +00:00
Code Issues Projects Releases Wiki Activity

Added manual mode and HTTPS support

Browse Source
This commit is contained in:
simon
2018-02-07 19:51:06 -05:00
parent 8e1f4543fd
commit a2f671f0f5
13 changed files with 217 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
spec/RedditBot_spec.py
View File

@@ -39,56 +39,42 @@ class TaskQueueTest(TestCase):
if os.path.isfile("task_queue_test.txt"):
os.remove("task_queue_test.txt")
def test_push_pop_test(self):
if os.path.isfile("task_queue_test.txt"):
os.remove("task_queue_test.txt")
tq = TaskQueue("task_queue_test.txt")
tq.push(CrawTask("http://awebsite.com/", "postid", "a title"))
task1 = tq.pop()
self.assertEqual(tq.pop(), None)
self.assertEqual(task1.url, "http://awebsite.com/")
self.assertEqual(task1.post_id, "postid")
def test_persistence(self):
if os.path.isfile("task_queue_test.txt"):
os.remove("task_queue_test.txt")
tq = TaskQueue("task_queue_test.txt")
tq.push(CrawTask("http://awebsite.com/", "postid", "a title"))
tq2 = TaskQueue("task_queue_test.txt")
task = tq2.pop()
self.assertEqual(task.url, "http://awebsite.com/")
self.assertEqual(task.post_id, "postid")
def test_multiple_tasks(self):
if os.path.isfile("task_queue_test.txt"):
os.remove("task_queue_test.txt")
tq = TaskQueue("task_queue_test.txt")
tq.push(CrawTask("http://awebsite.com/", "postid", "a title"))
tq.push(CrawTask("http://awebsite.com/", "postid", "a title"))
tq.push(CrawTask("http://awebsite.com/", "postid", "a title"))
self.assertIsNotNone(tq.pop())
self.assertIsNotNone(tq.pop())
self.assertIsNotNone(tq.pop())
self.assertIsNone(tq.pop())
def test_is_queued(self):
if os.path.isfile("task_queue_test.txt"):
os.remove("task_queue_test.txt")
tq = TaskQueue("task_queue_test.txt")
tq.push(CrawTask("http://awebsite.com/", "postid", "a title"))
self.assertTrue(tq.is_queued("postid"))
self.assertFalse(tq.is_queued("123456"))
# def test_push_pop_test(self):
#
# if os.path.isfile("task_queue_test.txt"):
# os.remove("task_queue_test.txt")
#
# tq = TaskQueue("task_queue_test.txt")
# tq.push(CrawTask(dict()))
#
# task1 = tq.pop()
#
# self.assertEqual(tq.pop(), None)
# self.assertEqual(task1.submission.url, "http://awebsite.com/")
# self.assertEqual(task1.submission.post_id, "postid")
#
# def test_multiple_tasks(self):
# if os.path.isfile("task_queue_test.txt"):
# os.remove("task_queue_test.txt")
#
# tq = TaskQueue("task_queue_test.txt")
#
# tq.push(CrawTask(dict()))
# tq.push(CrawTask(dict()))
# tq.push(CrawTask(dict()))
#
# self.assertIsNotNone(tq.pop())
# self.assertIsNotNone(tq.pop())
# self.assertIsNotNone(tq.pop())
# self.assertIsNone(tq.pop())
#
# def test_is_queued(self):
# if os.path.isfile("task_queue_test.txt"):
# os.remove("task_queue_test.txt")
#
# tq = TaskQueue("task_queue_test.txt")
#
# tq.push(CrawTask({id: "postid"}))
#
# self.assertTrue(tq.is_queued("postid"))
# self.assertFalse(tq.is_queued("123456"))

4
spec/ReportBuilder_spec.py
View File

@@ -1,6 +1,6 @@
import pickle
from unittest import TestCase
from reports import ReportBuilder
from crawl_report import ReportBuilder
class ReportBuilderTest(TestCase):
@@ -19,7 +19,7 @@ class ReportBuilderTest(TestCase):
def test_total_size_formatted(self):
result = self.report_builder.get_total_size_formatted()
self.assertEqual(result, "426.74 GB (426737457589 bytes)")
self.assertEqual(result, "426.74 GB")
def test_ext_counts(self):

4
spec/ReportSaver_spec.py
View File

@@ -1,6 +1,6 @@
import pickle
from unittest import TestCase
from reports import ReportSaver, ReportBuilder
from crawl_report import ReportSaver, ReportBuilder
import json
@@ -10,7 +10,7 @@ class ReportSaverTest(TestCase):
with open("test_report.pkl", 'rb') as f:
self.files = pickle.load(f)
self.report_saver = ReportSaver(self.files, ReportBuilder(self.files, "https://server.elscione.com/"))
self.report_saver = ReportSaver(self.files, "", ReportBuilder(self.files,"https://server.elscione.com/"))
with open("test_report.json", 'r') as f:
self.expected_json = f.read()