From a6ede3814bbea06672a61235cfa973c56923b466 Mon Sep 17 00:00:00 2001 From: simon987 Date: Mon, 28 Dec 2020 11:37:26 -0500 Subject: [PATCH] better resume tweaks --- scanner.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scanner.py b/scanner.py index 927c620..a96e219 100644 --- a/scanner.py +++ b/scanner.py @@ -114,11 +114,11 @@ class ParlerScanner: for item in j[items_key]: yield item, items_key + self._state.set_resume_key(resume_endpoint, resume_id, current_key) + if j["last"]: - self._state.set_resume_key(resume_endpoint, resume_id, None) break - self._state.set_resume_key(resume_endpoint, resume_id, current_key) current_key = j["next"] def user_followers(self, api, user_id): @@ -160,7 +160,7 @@ class ParlerScanner: def post_comments(self, api, post_id): for item, key in self._iterate_endpoint( func=api.comments_api.get_comments, - params={"id": post_id, "reverse": True}, + params={"id": post_id, "reverse": "true"}, resume_endpoint="comments", resume_id=post_id, # Also available: "users", "post", "postRefs" @@ -186,9 +186,7 @@ class ParlerScanner: return ParlerProfile(data=json.loads(r.content.decode('utf-8', 'ignore')), int_id=int_id) def user_ids(self, api): - current_key = 0 - - for int_id in range(current_key, 15000000): + for int_id in range(0, 15000000): if not self._state.has_visited_user(int_id): user_id = self._get_user_id_hash(api, int_id) if user_id: