Slots can be updated without removing & adding

app.py

@@ -447,10 +447,24 @@ def admin_delete_crawl_server(server_id):
         abort(403)
 
 
+@app.route("/crawl_server/<int:server_id>/update", methods=["POST"])
+def admin_update_crawl_server(server_id):
+
+    crawl_servers = db.get_crawl_servers()
+    for server in crawl_servers:
+        if server.id == server_id:
+
+            new_slots = request.form.get("slots") if "slots" in request.form else server.slots
+            new_name = request.form.get("name") if "name" in request.form else server.name
+            new_url = request.form.get("url") if "url" in request.form else server.url
+
+            db.update_crawl_server(server_id, new_url, new_name, new_slots)
+            flash("Updated crawl server", "success")
+            return redirect("/dashboard")
+
+    flash("Couldn't find crawl server with this id: " + str(server_id), "danger")
+    return redirect("/dashboard")
+
+
 if __name__ == '__main__':
-    if config.USE_SSL:
+    app.run("0.0.0.0", port=12345, threaded=True)
-        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-        context.load_cert_chain('certificates/cert.pem', 'certificates/privkey.pem')
-        app.run("0.0.0.0", port=12345, ssl_context=context, threaded=True)
-    else:
-        app.run("0.0.0.0", port=12345, threaded=True)

crawl_server/crawler.py

@@ -163,14 +163,6 @@ class RemoteDirectoryCrawler:
                 directory.close()
                 in_q.put(path)
                 break
-            except TimeoutError:
-                if timeout_retries > 0:
-                    timeout_retries -= 1
-                    # TODO: Remove debug info
-                    print("TIMEOUT, " + str(timeout_retries) + " retries left")
-                    in_q.put(path)
-                else:
-                    print("Dropping listing for " + path)
             finally:
                 in_q.task_done()
 

crawl_server/remote_http.py

@@ -64,7 +64,9 @@ class HttpDirectory(RemoteDirectory):
         "?C=D&O=A",
         "?C=N;O=D",
         "?C=M;O=A",
+        "?C=M&O=D",
         "?C=S;O=A",
+        "?C=S&O=D",
         "?C=D;O=A",
         "?MA",
         "?SA",

database.py

@@ -304,6 +304,14 @@ class Database:
 
             return [task.CrawlServer(r[0], r[1], r[2], r[3], r[4]) for r in cursor.fetchall()]
 
+    def update_crawl_server(self, server_id, url, name, slots):
+
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+
+            cursor.execute("UPDATE CrawlServer SET url=?, name=?, slots=? WHERE id=?", (url, name, slots, server_id))
+            conn.commit()
+
 
 
 

templates/dashboard.html

@@ -22,7 +22,7 @@
                         <tr>
                             <td>{{ server.url }}</td>
                             <td>{{ server.name }}</td>
-                            <td>{{ server.slots }}</td>
+                            <td id="slots-{{ server.id }}" onclick="changeSlots({{ server.id }})">{{ server.slots }}</td>
                             <td><a class="btn btn-danger" href="/crawl_server/{{ server.id }}/delete">Delete</a></td>
                         </tr>
                     {% endfor %}
@@ -127,4 +127,32 @@
             </div>
         </div>
     </div>
+    <script>
+
+    function changeSlots(id) {
+
+        let slotsElem = document.getElementById("slots-" + id);
+        let parent = slotsElem.parentNode;
+
+        let td = document.createElement("td");
+        let form = document.createElement("form");
+        form.setAttribute("action", "/crawl_server/" + id + "/update");
+        form.setAttribute("method", "post");
+
+        let slotsInput = document.createElement("input");
+        slotsInput.setAttribute("class", "form-control");
+        slotsInput.setAttribute("name", "slots");
+        form.appendChild(slotsInput);
+        td.appendChild(form);
+
+        parent.insertBefore(td, slotsElem);
+        slotsElem.remove();
+
+        slotsInput.focus();
+        slotsInput.addEventListener("focusout", function () {
+            form.submit();
+        });
+    }
+
+    </script>
 {% endblock body %}