simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 14:59:02 +00:00
Code Issues Projects Releases Wiki Activity

Crawl tasks are now fetched by the crawlers instead of pushed by the server

Browse Source
This commit is contained in:
Simon
2018-07-14 17:31:18 -04:00
parent d9e9f53f92
commit fe1d29aaea
20 changed files with 376 additions and 749 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
templates/crawl_logs.html
View File

@@ -7,14 +7,13 @@
<table class="table table-striped">
<thead>
<tr>
<th>Server</th>
<th>Crawler</th>
<th>Website</th>
<th>Status code</th>
<th>File count</th>
<th>Start</th>
<th>End</th>
<th>Delta</th>
<th>Index</th>
</tr>
</thead>
@@ -25,10 +24,9 @@
<td><a href="/website/{{ task_result.website_id }}/">#{{ task_result.website_id }}</a></td>
<td>{{ task_result.status_code }}</td>
<td>{{ task_result.file_count }}</td>
<td>{{ task_result.start_time | datetime_format }}</td>
<td>{{ task_result.end_time | datetime_format }}</td>
<td>{{ task_result.start_time | int | datetime_format }}</td>
<td>{{ task_result.end_time | int | datetime_format }}</td>
<td>{{ ((task_result.end_time - task_result.start_time)) | int }} sec</td>
<td>{{ task_result.indexed_time | datetime_format }}</td>
</tr>
{% endfor %}
</tbody>

91
templates/dashboard.html
View File

@@ -7,57 +7,15 @@
<div class="card-header">Dashboard</div>
<div class="card-body">
<a href="/logs">Logs</a>
<br>
<hr>
<h3>Crawl servers</h3>
<table class="table table-striped">
<thead>
<tr>
<th>Url</th>
<th>Name</th>
<th>Slots</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for server in crawl_servers %}
<tr>
<td>{{ server.url }}</td>
<td>{{ server.name }}</td>
<td id="slots-{{ server.id }}" onclick="changeSlots({{ server.id }})">{{ server.slots }}</td>
<td><a class="btn btn-danger" href="/crawl_server/{{ server.id }}/delete">Delete</a></td>
</tr>
{% endfor %}
</tbody>
</table>
<form action="/crawl_server/add" method="post">
<div class="form-row">
<div class="col col-md-3">
<input class="form-control" name="url" placeholder="Url">
</div>
<div class="col col-md-3">
<input class="form-control" name="name" placeholder="Name">
</div>
<div class="col col-md-2">
<input class="form-control" name="token" placeholder="Token">
</div>
<div class="col col-md-2">
<input class="form-control" name="slots" placeholder="Slots" type="number">
</div>
<div class="col col-md-2">
<input type="submit" class="form-control btn btn-primary" value="Add server">
</div>
</div>
</form>
<a href="/logs">Logs</a>
<br>
<hr>
<h3>API Keys</h3>
<table class="table table-striped">
<thead>
<tr>
<th>Description</th>
<th>Key</th>
<th>Name</th>
<th>Token</th>
<th>Action</th>
</tr>
</thead>
@@ -65,7 +23,7 @@
<tbody>
{% for token in api_tokens %}
<tr>
<td>{{ token.description }}</td>
<td>{{ token.name }}</td>
<td><code>{{ token.token }}</code></td>
<td>
<form action="/del_token" method="post">
@@ -122,7 +80,8 @@
<hr>
<h3>Misc actions</h3>
<a class="btn btn-danger" href="/website/delete_empty">Delete websites with no associated files that are not queued</a>
<a class="btn btn-danger" href="/website/delete_empty">Delete websites with no associated files that are
not queued</a>
<a class="btn btn-danger" href="/website/redispatch_queued">Re-dispatch queued tasks</a>
<a class="btn btn-danger" href="/website/queue_empty">Re-queue websites with no associated files</a>
@@ -133,30 +92,30 @@
</div>
<script>
function changeSlots(id) {
function changeSlots(id) {
let slotsElem = document.getElementById("slots-" + id);
let parent = slotsElem.parentNode;
let slotsElem = document.getElementById("slots-" + id);
let parent = slotsElem.parentNode;
let td = document.createElement("td");
let form = document.createElement("form");
form.setAttribute("action", "/crawl_server/" + id + "/update");
form.setAttribute("method", "post");
let td = document.createElement("td");
let form = document.createElement("form");
form.setAttribute("action", "/crawl_server/" + id + "/update");
form.setAttribute("method", "post");
let slotsInput = document.createElement("input");
slotsInput.setAttribute("class", "form-control");
slotsInput.setAttribute("name", "slots");
form.appendChild(slotsInput);
td.appendChild(form);
let slotsInput = document.createElement("input");
slotsInput.setAttribute("class", "form-control");
slotsInput.setAttribute("name", "slots");
form.appendChild(slotsInput);
td.appendChild(form);
parent.insertBefore(td, slotsElem);
slotsElem.remove();
parent.insertBefore(td, slotsElem);
slotsElem.remove();
slotsInput.focus();
slotsInput.addEventListener("focusout", function () {
form.submit();
});
}
slotsInput.focus();
slotsInput.addEventListener("focusout", function () {
form.submit();
});
}
</script>
{% endblock body %}

3
templates/home.html
View File

@@ -11,9 +11,6 @@
{% if stats and stats["total_size"] %}
<p class="lead">{{ stats["total_count"] }} files totalling
~{{ stats["total_size"] | filesizeformat }} from {{ stats["website_count"] }} websites</p>
{% if current_websites %}
<p>Currently indexing <code>{{ current_websites }}</code><span class="vim-caret">&nbsp;</span> </p>
{% endif %}
{% else %}
<p class="lead">We're currently experiencing a high volume of traffic. The search function
may be unresponsive.</p>

4
templates/stats.html
View File

@@ -100,13 +100,13 @@
{% endfor %}
</tr>
<tr>
<th>File crawled</th>
<th>Files crawled</th>
{% for server in crawl_server_stats %}
<td>{{ crawl_server_stats[server].file_count }}</td>
{% endfor %}
</tr>
<tr>
<th>File crawled average</th>
<th>Files crawled average</th>
{% for server in crawl_server_stats %}
<td>{{ crawl_server_stats[server].file_count_avg | round(2) }} per task</td>
{% endfor %}