Decentralised crawling should work in theory + temporary fix for going further than the maximum 10k results elasticsearch allows by default

init_script.sql

@@ -23,3 +23,11 @@ CREATE TABLE BlacklistedWebsite (
   id INTEGER PRIMARY KEY NOT NULL,
   url TEXT
 );
+
+CREATE TABLE CrawlServer (
+  id INTEGER PRIMARY KEY NOT NULL,
+  url TEXT,
+  name TEXT,
+  token TEXT,
+  slots INTEGER
+)