simon987/parler_feed
1
0
Fork 0
mirror of https://github.com/simon987/parler_feed.git synced 2025-04-20 10:46:43 +00:00
Code Issues Projects Releases Wiki Activity

better resume tweaks

Browse Source
This commit is contained in:
simon987 2020-12-28 11:37:26 -05:00
parent de97e7bc23
commit a6ede3814b
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scanner.py
View File

@ -114,11 +114,11 @@ class ParlerScanner:
for item in j[items_key]: for item in j[items_key]:
yield item, items_key yield item, items_key
self._state.set_resume_key(resume_endpoint, resume_id, current_key)
if j["last"]: if j["last"]:
self._state.set_resume_key(resume_endpoint, resume_id, None)
break break
self._state.set_resume_key(resume_endpoint, resume_id, current_key)
current_key = j["next"] current_key = j["next"]
def user_followers(self, api, user_id): def user_followers(self, api, user_id):
@ -160,7 +160,7 @@ class ParlerScanner:
def post_comments(self, api, post_id): def post_comments(self, api, post_id):
for item, key in self._iterate_endpoint( for item, key in self._iterate_endpoint(
func=api.comments_api.get_comments, func=api.comments_api.get_comments,
params={"id": post_id, "reverse": True}, params={"id": post_id, "reverse": "true"},
resume_endpoint="comments", resume_endpoint="comments",
resume_id=post_id, resume_id=post_id,
# Also available: "users", "post", "postRefs" # Also available: "users", "post", "postRefs"
@ -186,9 +186,7 @@ class ParlerScanner:
return ParlerProfile(data=json.loads(r.content.decode('utf-8', 'ignore')), int_id=int_id) return ParlerProfile(data=json.loads(r.content.decode('utf-8', 'ignore')), int_id=int_id)
def user_ids(self, api): def user_ids(self, api):
current_key = 0 for int_id in range(0, 15000000):
for int_id in range(current_key, 15000000):
if not self._state.has_visited_user(int_id): if not self._state.has_visited_user(int_id):
user_id = self._get_user_id_hash(api, int_id) user_id = self._get_user_id_hash(api, int_id)
if user_id: if user_id: