simon987/Simple-Incremental-Search-Tool
1
0
Fork 0
mirror of https://github.com/simon987/Simple-Incremental-Search-Tool.git synced 2025-04-15 16:26:45 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2 from simon987/french-translation

Browse Source 
bug fixes
This commit is contained in:
Simon Fortier 2019-03-10 15:20:50 -04:00 committed by GitHub
commit 7ccc1f14d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 412 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
README.md
View File

@ -1,6 +1,7 @@
# Simple incremental search tool
Work in progress! Shouldn't be used in production environnments.
Portable search tool for local files using Elasticsearch.
### Features
* Incremental search (Search as you type)
@ -19,37 +20,21 @@ Work in progress! Shouldn't be used in production environnments.
# Installation
Java and python3 are required. To parse video and audio files, `ffmpeg` needs to be installed
Java and python3 are required.
Once the web server is running, you can connect to the search interface by typing `localhost:8080` in your browser.
## Setup on Windows
## Setup on Windows/Mac/linux (Python 3.5+)
* Download and install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch)
```bash
git clone https://github.com/simon987/Simple-Incremental-Search-Tool
cd Simple-Incremental-Search-Tool
```
[Download latest elasticsearch version](https://www.elastic.co/downloads/elasticsearch) and extract to `Simple-Incremental-Search-Tool\elasticsearch`
```bash
sudo pip3 install -r requirements.txt
python3 run.py
```
## Setup on Mac/linux
```bash
git clone https://github.com/simon987/Simple-Incremental-Search-Tool
cd Simple-Incremental-Search-Tool
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.4.zip
unzip elasticsearch-6.2.4.zip
rm elasticsearch-6.2.4.zip
mv elasticsearch-6.2.4 elasticsearch
sudo pip3 install -r requirements.txt
python3 run.py
```
## Running unit tests
```bash
python3 -m unittest
```
```

3
config.py
View File

@ -25,6 +25,9 @@ bcrypt_rounds = 14
# sqlite3 database path
db_path = "./local_storage.db"
# Set to true to allow guests to search any directory
allow_guests = False
try:
import cairosvg
cairosvg = True

7
crawler.py
View File

@ -12,7 +12,6 @@ from thumbnail import ThumbnailGenerator
from storage import Directory
import shutil
import config
from ctypes import c_char_p
class RunningTask:
@ -145,9 +144,9 @@ class TaskManager:
TextFileParser(chksum_calcs, int(directory.get_option("TextFileContentLength")), directory.path),
PictureFileParser(chksum_calcs, directory.path),
FontParser(chksum_calcs, directory.path),
PdfFileParser(chksum_calcs, int(directory.get_option("TextFileContentLength")), directory.path), # todo get content len from other opt
DocxParser(chksum_calcs, int(directory.get_option("TextFileContentLength")), directory.path), # todo get content len from other opt
EbookParser(chksum_calcs, int(directory.get_option("TextFileContentLength")), directory.path)], # todo get content len from other opt
PdfFileParser(chksum_calcs, int(directory.get_option("PdfFileContentLength")), directory.path),
DocxParser(chksum_calcs, int(directory.get_option("SpreadsheetContentLength")), directory.path),
EbookParser(chksum_calcs, int(directory.get_option("EbookContentLength")), directory.path)],
mime_guesser, self.indexer, directory.id)
c.crawl(directory.path, counter)

33
indexer.py
View File

@ -1,10 +1,9 @@
import json
import elasticsearch
from threading import Thread
import subprocess
import requests
import config
import platform
class Indexer:
@ -14,30 +13,12 @@ class Indexer:
self.index_name = index
self.es = elasticsearch.Elasticsearch()
try:
requests.head("http://localhost:9200")
except requests.exceptions.ConnectionError:
import time
t = Thread(target=Indexer.run_elasticsearch)
t.daemon = True
t.start()
time.sleep(15)
if self.es.indices.exists(self.index_name):
print("Index is already setup")
else:
print("First time setup...")
self.init()
@staticmethod
def run_elasticsearch():
if platform.system() == "Windows":
subprocess.Popen(["elasticsearch\\bin\\elasticsearch.bat"])
requests.head("http://localhost:9200")
if self.es.indices.exists(self.index_name):
print("Index is already setup")
else:
subprocess.Popen(["elasticsearch/bin/elasticsearch"])
print("First time setup...")
self.init()
@staticmethod
def create_bulk_index_string(docs: list, directory: int):

2
parsing.py
View File

@ -272,7 +272,7 @@ class TextFileParser(GenericFileParser):
"text/x-bibtex", "text/x-tcl", "text/x-c++", "text/x-shellscript", "text/x-msdos-batch",
"text/x-makefile", "text/rtf", "text/x-objective-c", "text/troff", "text/x-m4",
"text/x-lisp", "text/x-php", "text/x-gawk", "text/x-awk", "text/x-ruby", "text/x-po",
"text/x-makefile", "application/javascript", "application/rtf"
"text/x-makefile", "application/javascript", "application/rtf", "application/json",
]
def parse(self, full_path: str):

3
requirements.txt
View File

@ -16,4 +16,5 @@ html2text
docx2txt
xlrd
six
cairosvg
cairosvg
ffmpeg-python

415
run.py
View File

@ -1,17 +1,18 @@
from flask import Flask, render_template, request, redirect, flash, session, abort, send_file
from storage import Directory, Option, Task, User
from storage import LocalStorage, DuplicateDirectoryException, DuplicateUserException
from crawler import RunningTask, TaskManager
import json
import os
import shutil
import bcrypt
import config
import humanfriendly
from search import Search
from PIL import Image
from io import BytesIO
import bcrypt
import humanfriendly
from PIL import Image
from flask import Flask, render_template, request, redirect, flash, session, abort, send_file
import config
from crawler import TaskManager
from search import Search
from storage import Directory, Option, Task, User
from storage import LocalStorage, DuplicateDirectoryException, DuplicateUserException
app = Flask(__name__)
app.secret_key = "A very secret key"
@ -22,12 +23,10 @@ search = Search("changeme")
def get_dir_size(path):
size = 0
for root, dirs, files in os.walk(path):
for filename in files:
full_path = os.path.join(root, filename)
size += os.path.getsize(full_path)
@ -36,8 +35,11 @@ def get_dir_size(path):
@app.route("/user/<user>")
def user_manage(user):
return user
if "admin" in session and session["admin"]:
return render_template("user_manage.html", directories=storage.dirs(), user=storage.users()[user])
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/logout")
@ -57,7 +59,6 @@ def login():
session["username"] = username
session["admin"] = storage.users()[username].admin
print(session["admin"])
flash("Successfully logged in", "success")
else:
flash("Invalid username or password", "danger")
@ -67,35 +68,108 @@ def login():
@app.route("/user")
def user_page():
admin_account_present = False
return render_template("user.html", users=storage.users())
for user in storage.users():
if storage.users()[user].admin:
admin_account_present = True
break
if not admin_account_present or ("admin" in session and session["admin"]):
return render_template("user.html", users=storage.users(), admin_account_present=admin_account_present)
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/user/<username>/set_access")
def user_set_access(username):
if "admin" in session and session["admin"]:
dir_id = request.args["dir_id"]
user = storage.users()[username]
if request.args["access"] == "1":
user.readable_directories.add(int(dir_id))
else:
if int(dir_id) in user.readable_directories:
user.readable_directories.remove(int(dir_id))
storage.update_user(user)
flash("Permissions mises à jour", "success")
return redirect("/user/" + username)
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/user/<username>/set_admin")
def user_set_admin(username):
if "admin" in session and session["admin"]:
user = storage.users()[username]
if user.username == session["username"]:
flash("You cannot modifiy your own account", "warning")
else:
user.admin = request.args["admin"] == "1"
storage.update_user(user)
flash("Permissions updated", "success")
return redirect("/user/" + username)
@app.route("/user/add", methods=['POST'])
def user_add():
admin_account_present = False
username = request.form["username"]
password = bcrypt.hashpw(request.form["password"].encode("utf-8"), bcrypt.gensalt(config.bcrypt_rounds))
is_admin = True if "is_admin" in request.form else False
for user in storage.users():
if storage.users()[user].admin:
admin_account_present = True
break
try:
storage.save_user(User(username, password, is_admin))
flash("Created new user", "success")
except DuplicateUserException:
flash("<strong>Couldn't create user</strong> Make sure that the username is unique", "danger")
if not admin_account_present or ("admin" in session and session["admin"]):
username = request.form["username"]
password = bcrypt.hashpw(request.form["password"].encode("utf-8"), bcrypt.gensalt(config.bcrypt_rounds))
is_admin = True if "is_admin" in request.form else False
return redirect("/user")
try:
storage.save_user(User(username, password, is_admin))
flash("Created new user", "success")
except DuplicateUserException:
flash("<strong>Couldn't create user</strong> "
"Make sure that the username is unique", "danger")
return redirect("/user")
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/user/<username>/del")
def user_del(username):
if "admin" in session and session["admin"]:
if session["username"] == username:
flash("You cannot delete your own account", "warning")
return redirect("/user/" + username)
else:
storage.remove_user(username)
flash("User deleted", "success")
return redirect("/user")
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/suggest")
def suggest():
return json.dumps(search.suggest(request.args.get("prefix")))
@app.route("/document/<doc_id>")
def document(doc_id):
doc = search.get_doc(doc_id)["_source"]
directory = storage.dirs()[doc["directory"]]
@ -106,7 +180,6 @@ def document(doc_id):
@app.route("/dl/<doc_id>")
def file(doc_id):
doc = search.get_doc(doc_id)["_source"]
directory = storage.dirs()[doc["directory"]]
@ -121,7 +194,6 @@ def file(doc_id):
@app.route("/file/<doc_id>")
def download(doc_id):
doc = search.get_doc(doc_id)["_source"]
directory = storage.dirs()[doc["directory"]]
extension = "" if doc["extension"] is None or doc["extension"] == "" else "." + doc["extension"]
@ -135,7 +207,6 @@ def download(doc_id):
@app.route("/thumb/<doc_id>")
def thumb(doc_id):
doc = search.get_doc(doc_id)
if doc is not None:
@ -145,14 +216,12 @@ def thumb(doc_id):
if os.path.isfile(tn_path):
return send_file(tn_path)
else:
print("tn not found")
default_thumbnail = BytesIO()
Image.new("RGB", (255, 128), (0, 0, 0)).save(default_thumbnail, "JPEG")
default_thumbnail.seek(0)
return send_file(default_thumbnail, "image/jpeg")
else:
print("doc is none")
default_thumbnail = BytesIO()
Image.new("RGB", (255, 128), (0, 0, 0)).save(default_thumbnail, "JPEG")
default_thumbnail.seek(0)
@ -161,11 +230,14 @@ def thumb(doc_id):
@app.route("/")
def search_page():
mime_map = search.get_mime_map()
mime_map.append({"id": "any", "text": "Any"})
mime_map.append({"id": "any", "text": "All"})
return render_template("search.html", directories=storage.dirs(), mime_map=mime_map)
directories = [storage.dirs()[x] for x in get_allowed_dirs(session["username"] if "username" in session else None)]
return render_template("search.html",
directories=directories,
mime_map=mime_map)
@app.route("/list")
@ -173,9 +245,19 @@ def search_liste_page():
return render_template("searchList.html")
def get_allowed_dirs(username):
if config.allow_guests:
return [x for x in storage.dirs() if x.enabled]
else:
if username:
user = storage.users()[username]
return [x for x in storage.dirs() if storage.dirs()[x].enabled and x in user.readable_directories]
else:
return list()
@app.route("/search", methods=['POST'])
def search_route():
query = request.json["q"]
query = "" if query is None else query
@ -186,19 +268,8 @@ def search_route():
directories = request.json["directories"]
# Remove disabled & non-existing directories
for search_directory in directories:
directory_exists = False
for dir_id in storage.dirs():
if search_directory == dir_id:
directory_exists = True
if not storage.dirs()[dir_id].enabled:
directories.remove(search_directory)
break
if not directory_exists:
directories.remove(search_directory)
allowed_dirs = get_allowed_dirs(session["username"] if "username" in session else None)
directories = [x for x in directories if x in allowed_dirs]
path = request.json["path"]
@ -209,7 +280,6 @@ def search_route():
@app.route("/scroll")
def scroll_route():
scroll_id = request.args.get("scroll_id")
page = search.scroll(scroll_id)
@ -219,184 +289,237 @@ def scroll_route():
@app.route("/directory")
def dir_list():
return render_template("directory.html", directories=storage.dirs())
if "admin" in session and session["admin"]:
return render_template("directory.html", directories=storage.dirs())
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/directory/add")
def directory_add():
if "admin" in session and session["admin"]:
path = request.args.get("path")
name = request.args.get("name")
path = request.args.get("path")
name = request.args.get("name")
if path is not None and name is not None:
d = Directory(path, True, [], name)
if path is not None and name is not None:
d = Directory(path, True, [], name)
try:
d.set_default_options()
storage.save_directory(d)
flash("<strong>Directory created</strong>", "success")
except DuplicateDirectoryException:
flash("<strong>The directory couldn't be created</strong> Make sure to chose a unique name",
"danger")
try:
d.set_default_options()
storage.save_directory(d)
flash("<strong>Created directory</strong>", "success")
except DuplicateDirectoryException:
flash("<strong>Couldn't create directory</strong> Make sure that the path is unique", "danger")
return redirect("/directory")
return redirect("/directory")
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/directory/<int:dir_id>")
def directory_manage(dir_id):
if "admin" in session and session["admin"]:
directory = storage.dirs()[dir_id]
tn_size = get_dir_size("static/thumbnails/" + str(dir_id))
tn_size_formatted = humanfriendly.format_size(tn_size)
directory = storage.dirs()[dir_id]
tn_size = get_dir_size("static/thumbnails/" + str(dir_id))
tn_size_formatted = humanfriendly.format_size(tn_size)
return render_template("directory_manage.html", directory=directory, tn_size=tn_size,
tn_size_formatted=tn_size_formatted)
return render_template("directory_manage.html", directory=directory, tn_size=tn_size,
tn_size_formatted=tn_size_formatted)
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/directory/<int:dir_id>/update")
def directory_update(dir_id):
if "admin" in session and session["admin"]:
directory = storage.dirs()[dir_id]
directory = storage.dirs()[dir_id]
name = request.args.get("name")
name = directory.name if name is None else name
name = request.args.get("name")
name = directory.name if name is None else name
enabled = request.args.get("enabled")
enabled = directory.enabled if enabled is None else int(enabled)
enabled = request.args.get("enabled")
enabled = directory.enabled if enabled is None else int(enabled)
path = request.args.get("path")
path = directory.path if path is None else path
path = request.args.get("path")
path = directory.path if path is None else path
# Only name and enabled status can be updated
updated_dir = Directory(path, enabled, directory.options, name)
updated_dir.id = dir_id
# Only name and enabled status can be updated
updated_dir = Directory(path, enabled, directory.options, name)
updated_dir.id = dir_id
try:
storage.update_directory(updated_dir)
flash("<strong>Updated directory</strong>", "success")
try:
storage.update_directory(updated_dir)
flash("<strong>Updated directory</strong>", "success")
except DuplicateDirectoryException:
flash("<strong>The directory couldn't be updated</strong> Make the that the path is unique",
"danger")
except DuplicateDirectoryException:
flash("<strong>Couldn't update directory</strong> Make sure that the path is unique", "danger")
return redirect("/directory/" + str(dir_id))
return redirect("/directory/" + str(dir_id))
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/directory/<int:dir_id>/update_opt")
def directory_update_opt(dir_id):
if "admin" in session and session["admin"]:
opt_id = request.args.get("id")
opt_key = request.args.get("key")
opt_value = request.args.get("value")
opt_id = request.args.get("id")
opt_key = request.args.get("key")
opt_value = request.args.get("value")
storage.update_option(Option(opt_key, opt_value, dir_id, opt_id))
storage.update_option(Option(opt_key, opt_value, dir_id, opt_id))
return redirect("/directory/" + str(dir_id))
return redirect("/directory/" + str(dir_id))
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/directory/<int:dir_id>/del")
def directory_del(dir_id):
if "admin" in session and session["admin"]:
search.delete_directory(dir_id)
if os.path.exists("static/thumbnails/" + str(dir_id)):
shutil.rmtree("static/thumbnails/" + str(dir_id))
search.delete_directory(dir_id)
if os.path.exists("static/thumbnails/" + str(dir_id)):
shutil.rmtree("static/thumbnails/" + str(dir_id))
storage.remove_directory(dir_id)
flash("<strong>Deleted folder</strong>", "success")
storage.remove_directory(dir_id)
flash("<strong>Deleted directory</strong>", "success")
return redirect("/directory")
return redirect("/directory")
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/directory/<int:dir_id>/reset")
def directory_reset(dir_id):
directory = storage.dirs()[dir_id]
if "admin" in session and session["admin"]:
directory = storage.dirs()[dir_id]
for opt in directory.options:
storage.del_option(opt.id)
for opt in directory.options:
storage.del_option(opt.id)
directory.set_default_options()
directory.set_default_options()
for opt in directory.options:
opt.dir_id = dir_id
storage.save_option(opt)
for opt in directory.options:
opt.dir_id = dir_id
storage.save_option(opt)
storage.dir_cache_outdated = True
storage.dir_cache_outdated = True
search.delete_directory(dir_id)
search.delete_directory(dir_id)
flash("<strong>Reset directory options to default settings</strong>", "success")
return redirect("directory/" + str(dir_id))
flash("<strong>Reset directory options</strong>", "success")
return redirect("directory/" + str(dir_id))
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/task")
def task():
return render_template("task.html", tasks=storage.tasks(), directories=storage.dirs(),
task_list=json.dumps(list(storage.tasks().keys())))
# return render_template("task.html", tasks=storage.tasks(), directories=storage.dirs())
if "admin" in session and session["admin"]:
return render_template("task.html", tasks=storage.tasks(), directories=storage.dirs(),
task_list=json.dumps(list(storage.tasks().keys())))
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/task/current")
def get_current_task():
if "admin" in session and session["admin"]:
if tm and tm.current_task:
return tm.current_task.to_json()
if tm and tm.current_task:
return tm.current_task.to_json()
else:
return ""
else:
return ""
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/task/add")
def task_add():
type = request.args.get("type")
directory = request.args.get("directory")
if "admin" in session and session["admin"]:
task_type = request.args.get("type")
directory = request.args.get("directory")
storage.save_task(Task(type, directory))
if task_type not in ("1", "2"):
flash("Please choose a task type", "danger")
return redirect("/task")
return redirect("/task")
if directory.isdigit() and int(directory) in storage.dirs():
storage.save_task(Task(task_type, directory))
else:
flash("You must choose a directory", "danger")
return redirect("/task")
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/task/<int:task_id>/del")
def task_del(task_id):
storage.del_task(task_id)
if "admin" in session and session["admin"]:
storage.del_task(task_id)
if tm.current_task is not None and task_id == tm.current_task.task.id:
tm.cancel_task()
if tm.current_task is not None and task_id == tm.current_task.task.id:
tm.cancel_task()
return redirect("/task")
return redirect("/task")
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/reset_es")
def reset_es():
if "admin" in session and session["admin"]:
flash("Elasticsearch index has been reset. Modifications made in <b>config.py</b> have been applied.",
"success")
flash("Elasticsearch index has been reset. Modifications made in <b>config.py</b> have been applied.", "success")
tm.indexer.init()
if os.path.exists("static/thumbnails"):
shutil.rmtree("static/thumbnails")
tm.indexer.init()
if os.path.exists("static/thumbnails"):
shutil.rmtree("static/thumbnails")
return redirect("/dashboard")
return redirect("/dashboard")
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
@app.route("/dashboard")
def dashboard():
if "admin" in session and session["admin"]:
tn_sizes = {}
tn_size_total = 0
for directory in storage.dirs():
tn_size = get_dir_size("static/thumbnails/" + str(directory))
tn_size_formatted = humanfriendly.format_size(tn_size)
tn_sizes = {}
tn_size_total = 0
for directory in storage.dirs():
tn_size = get_dir_size("static/thumbnails/" + str(directory))
tn_size_formatted = humanfriendly.format_size(tn_size)
tn_sizes[directory] = tn_size_formatted
tn_size_total += tn_size
tn_sizes[directory] = tn_size_formatted
tn_size_total += tn_size
tn_size_total_formatted = humanfriendly.format_size(tn_size_total)
tn_size_total_formatted = humanfriendly.format_size(tn_size_total)
return render_template("dashboard.html", version=config.VERSION, tn_sizes=tn_sizes,
tn_size_total=tn_size_total_formatted,
doc_size=humanfriendly.format_size(search.get_doc_size()),
doc_count=search.get_doc_count(),
db_path=config.db_path,
elasticsearch_url=config.elasticsearch_url,
index_size=humanfriendly.format_size(search.get_index_size()))
return render_template("dashboard.html", version=config.VERSION, tn_sizes=tn_sizes,
tn_size_total=tn_size_total_formatted,
doc_size=humanfriendly.format_size(search.get_doc_size()),
doc_count=search.get_doc_count(),
db_path=config.db_path,
elasticsearch_url=config.elasticsearch_url,
index_size=humanfriendly.format_size(search.get_index_size()))
else:
flash("You are not authorized to access this page", "warning")
return redirect("/")
if __name__ == "__main__":

4
static/css/bootstrap.min.css vendored
View File

File diff suppressed because one or more lines are too long

6
static/js/search.js
View File

@ -173,8 +173,8 @@ function gifOver(thumbnail, documentId) {
function downloadPopover(element, documentId) {
element.setAttribute("data-content",
'<a class="btn btn-sm btn-primary" href="/dl/'+ documentId +'"><i class="fas fa-download"></i> Download</a>' +
'<a class="btn btn-sm btn-success" style="margin-left:3px;" href="/file/'+ documentId + '" target="_blank"><i class="fas fa-eye"></i> View</a>');
'<a class="btn btn-sm btn-primary" href="/dl/'+ documentId +'"><i class="fas fa-download"></i> Télécharger</a>' +
'<a class="btn btn-sm btn-success" style="margin-left:3px;" href="/file/'+ documentId + '" target="_blank"><i class="fas fa-eye"></i> Consulter</a>');
element.setAttribute("data-toggle", "popover");
element.addEventListener("mouseover", function() {
element.focus();
@ -592,4 +592,4 @@ document.getElementById("pathBar").addEventListener("keyup", function () {
searchQueued = true;
});
window.setInterval(search, 150);
window.setInterval(search, 300);

20
storage.py
View File

@ -7,6 +7,9 @@ import config
class CheckSumCalculator:
def __init__(self):
pass
def checksum(self, string: str):
return flask_bcrypt.generate_password_hash(string, config.bcrypt_rounds)
@ -28,6 +31,7 @@ class User:
self.username = username
self.hashed_password = hashed_password
self.admin = admin
self.readable_directories = set()
class Option:
@ -224,7 +228,15 @@ class LocalStorage:
db_users = c.fetchall()
for db_user in db_users:
self.cached_users[db_user[0]] = User(db_user[0], b"", bool(db_user[1]))
user = User(db_user[0], b"", bool(db_user[1]))
c.execute("SELECT username, directory_id FROM User_canRead_Directory WHERE username=?", (db_user[0],))
db_accesses = c.fetchall()
for db_access in db_accesses:
user.readable_directories.add(db_access[1])
self.cached_users[db_user[0]] = user
conn.close()
@ -256,6 +268,12 @@ class LocalStorage:
c = conn.cursor()
c.execute("UPDATE User SET is_admin=? WHERE username=?", (user.admin, user.username))
c.execute("DELETE FROM User_canRead_Directory WHERE username=?", (user.username, ))
conn.commit()
for access in user.readable_directories:
c.execute("INSERT INTO User_canRead_Directory VALUES (?,?)", (user.username, access))
c.close()
conn.commit()
conn.close()

6
templates/dashboard.html
View File

@ -5,7 +5,7 @@
<div class="container">
<div class="card">
<div class="card-header">FSE Information</div>
<div class="card-header">Global information</div>
<div class="card-body">
<table class="info-table table-hover table-striped">
<tbody>
@ -26,7 +26,7 @@
<td><pre>{{ doc_size }}</pre></td>
</tr>
<tr>
<th>Total index size</th>
<th>total index size</th>
<td><pre>{{ index_size }}</pre></td>
</tr>
<tr>
@ -52,7 +52,7 @@
<div class="card-header">Actions</div>
<div class="card-body">
<button class="btn btn-danger" onclick="resetAll()">
<i class="fas fa-exclamation-triangle"></i> Reset elasticsearch index
<i class="fas fa-exclamation-triangle"></i> Reset Elasticsearch
</button>
</div>
</div>

18
templates/directory.html
View File

@ -1,14 +1,14 @@
{% extends "layout.html" %}
{% set active_page = "directory" %}
{% block title %}An Excellent title{% endblock title %}
{% block title %}Directory list{% endblock title %}
{% block body %}
<div class="container">
{# Add directory form #}
<div class="card">
<div class="card-header">An excellent form</div>
<div class="card-header">Add a directory</div>
<div class="card-body">
<form method="GET" action="/directory/add">
@ -18,7 +18,7 @@
<div class="form-group">
<input type="text" class="form-control" placeholder="Absolute path" name="path">
</div>
<button type="submit" class="btn btn-success"><i class="fas fa-plus"></i> Add Directory</button>
<button type="submit" class="btn btn-success"><i class="fas fa-plus"></i> Add directory</button>
</form>
</div>
@ -26,16 +26,15 @@
{# List of directories #}
<div class="card">
<div class="card-header">An excellent list</div>
<div class="card-header">Directory list</div>
<div class="card-body">
<table class="info-table table-hover table-striped">
<thead>
<tr>
<th>Display Name</th>
<th>Path</th>
<th>Enabled</th>
<th>Last indexed</th>
<th>Nom</th>
<th>Chemin</th>
<th>Activé</th>
<th>Action</th>
</tr>
</thead>
@ -45,8 +44,7 @@
<td>{{ directories[dir].name }}</td>
<td style="word-break: break-all"><pre>{{ directories[dir].path }}</pre></td>
<td><i class="far {{ "fa-check-square" if directories[dir].enabled else "fa-square" }}"></i></td>
<td>2018-02-21</td>
<td><a href="directory/{{ dir }}" class="btn btn-primary"><i class="fas fa-cog"></i> Manage</a> </td>
<td><a href="directory/{{ dir }}" class="btn btn-primary"><i class="fas fa-cog"></i> </a> Manage</td>
</tr>
{% endfor %}
</tbody>

24
templates/directory_manage.html
View File

@ -1,7 +1,7 @@
{% extends "layout.html" %}
{% extends "layout.html" %}
{% set active_page = "directory" %}
{% block title %}An excellent title{% endblock title %}
{% block title %}{{ directory.name }}{% endblock title %}
{% block body %}
@ -75,7 +75,7 @@
</tr>
<tr onclick="modifyPath()">
<th style="width: 20%">Path</th>Task
<th style="width: 20%">Path</th>
<td>
<pre id="path" title="Click to update">{{ directory.path }}</pre>
</td>
@ -84,7 +84,8 @@
<tr>
<th style="width: 20%">Enabled</th>
<td>
<form action="/directory/{{ directory.id }}/update" style="display: inline;margin-left: 6px;">
<form action="/directory/{{ directory.id }}/update"
style="display: inline;margin-left: 6px;">
<input type="hidden" name="enabled" value="{{ "0" if directory.enabled else "1" }}">
<button class="btn btn-sm {{ "btn-danger" if directory.enabled else "btn-success" }}">
<i class="far {{ "fa-check-square" if directory.enabled else "fa-square" }}"></i>
@ -96,7 +97,9 @@
<tr>
<th>Thumbnail cache size</th>
<td><pre>{{ tn_size_formatted }} ({{ tn_size }} bytes)</pre></td>
<td>
<pre>{{ tn_size_formatted }} ({{ tn_size }} bytes)</pre>
</td>
</tr>
</table>
</div>
@ -112,7 +115,7 @@
<form action="/task/add" class="p-2">
<input type="hidden" value="1" name="type">
<input type="hidden" value="{{ directory.id }}" name="directory">
<button class="btn btn-primary" href="/task/" value="Generate thumbnails">
<button class="btn btn-primary" href="/task/">
<i class="fas fa-book"></i> Generate index
</button>
</form>
@ -120,7 +123,7 @@
<form action="/task/add" class="p-2">
<input type="hidden" value="2" name="type">
<input type="hidden" value="{{ directory.id }}" name="directory">
<button class="btn btn-primary" href="/task/" value="Generate thumbnails">
<button class="btn btn-primary" href="/task/">
<i class="far fa-images"></i> Generate thumbnails
</button>
</form>
@ -140,7 +143,8 @@
</div>
<div class="card">
<div class="card-header">Options <a href="#" style="float:right">Learn more <i class="fas fa-external-link-alt"></i></a></div>
<div class="card-header">Options <a href="#" style="float:right">Learn more <i
class="fas fa-external-link-alt"></i></a></div>
<div class="card-body">
<table class="info-table table-striped table-hover">
<thead>
@ -154,7 +158,9 @@
<tr>
<td style="width: 30%"><span>{{ option.key }}</span></td>
<td onclick="modifyVal({{ option.id }}, '{{ option.key }}')" title="Click to update"><pre id="val-{{ option.id }}">{{ option.value }}</pre></td>
<td onclick="modifyVal({{ option.id }}, '{{ option.key }}')" title="Click to update">
<pre id="val-{{ option.id }}">{{ option.value }}</pre>
</td>
</tr>
{% endfor %}

2
templates/document.html
View File

@ -26,7 +26,7 @@
<hr>
<h3>Raw json</h3>
<h3>Raw JSON</h3>
<textarea class="form-control" style="min-height: 100px" readonly>{{ doc | tojson }}</textarea>
<hr>

71
templates/search-old.html
View File

@ -1,71 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<style>
.fit:hover {
-webkit-transform:scale(2.5);
-moz-transform:scale(2.5);
-ms-transform:scale(2.5);
-o-transform:scale(2.5);
transform:scale(2.5);
}
.fit {
width: 100%;
height: 100%;
}
.image-container{
width: 305px;
height: 300px;
background-color: #ccc;
overflow: hidden;
}
.doc-container {
height: 330px;
display: inline-block;
}
.doc-caption {
display: inline-block;
}
</style>
<div class="photos">
{% for doc in docs %}
{% if doc.type == "audio" %}
<div class="image-container">
<audio controls class="fit">
<!--<source src="files/{{doc.doc_id}}">-->
</audio>
</div>
{% else %}
<a href="/files/{{doc.doc_id}}">
<div class="doc-container">
<div class="image-container">
<img class="fit" src="/thumbs/{{doc.doc_id}}">
</div>
<span class="doc-caption" style="font-size: 8pt">{{doc.name}}</span>
</div>
</a>
{% endif %}
{% endfor %}
</div>
</body>
</html>

12
templates/search.html
View File

@ -34,11 +34,13 @@
<label for="directories" >Search in directories</label>
<select class="custom-select" id="directories" multiple size="6">
{% for dir_id in directories %}
{% if directories[dir_id].enabled %}
<option selected value="{{ directories[dir_id].id }}">{{ directories[dir_id].name }}</option>
{% endif %}
{% endfor %}
{% if directories | length > 0%}
{% for dir in directories %}
<option selected value="{{ dir.id }}">{{ dir.name }}</option>
{% endfor %}
{% else %}
<option disabled>There are no active directories which you have access to</option>
{% endif %}
</select>
</div>

4
templates/task.html
View File

@ -1,7 +1,7 @@
{% extends "layout.html" %}
{% set active_page = "task" %}
{% block title %}An excellent title{% endblock title %}
{% block title %}Tasks{% endblock title %}
{% block body %}
@ -49,7 +49,7 @@
<select title="Select task type" class="form-control" id="type" name="type">
<option hidden>Create task...</option>
<option value="1">Indexing</option>
<option value="2">Thumbnail Generation</option>
<option value="2">Thumnail Generation</option>
</select>
<select title="Select directory" class="form-control" id="directory" name="directory" >

18
templates/user.html
View File

@ -1,14 +1,18 @@
{% extends "layout.html" %}
{% set active_page = "user" %}
{% block title %}User list{% endblock title %}
{% block body %}
<div class="container">
<div class="card">
<div class="container"> <div class="card">
<div class="card-header">Create user</div>
<div class="card-body">
{% if not admin_account_present %}
<p>This page is unlocked because there are no admin accounts</p>
{% endif %}
<form method="POST" action="/user/add">
<div class="input-group form-group">
@ -24,7 +28,7 @@
<div class="form-group">
<input type="password" class="form-control" placeholder="Password" name="password">
</div>
<button type="submit" class="btn btn-success"><i class="fas fa-plus"></i> Add User</button>
<button type="submit" class="btn btn-success"><i class="fas fa-plus"></i> Add user</button>
</form>
</div>
</div>
@ -39,12 +43,10 @@
<th>User</th>
<th>Admin</th>
<th>Actions</th>
</tr>
</thead>
</tr> </thead>
<tbody>
{% for user in users %}
<tr>
<td>{{ user }}</td>
{% for user in users %} <tr>
<td style="width: 80%;">{% if session["username"] == user %}<b>{{ user }}{% else %}{{ user }}{% endif %}</b></td>
<td><i class="far {{ "fa-check-square" if users[user].admin else "fa-square" }}"></i></td>
<td><a href="/user/{{ user }}" class="btn btn-primary">Manage</a></td>
</tr>

52
templates/user_manage.html
View File

@ -1,33 +1,71 @@
{% extends "layout.html" %}
{% set active_page = "user" %}
{% block title %}Gestion utilisateur{% endblock title %}
{% block body %}
<div class="container">
<div class="card">
<div class="card-header">Directory permissions</div>
<div class="card-header">Manage permissions of <strong>{{ user.username }}</strong></div>
<div class="card-body">
<div class="row">
<div class="col">
<h5>Admin: </h5>
</div>
<div class="col">
<form action="/user/{{ user.username }}/set_admin" style="display: inline;margin-left: 6px;">
<input type="hidden" name="admin" value="{{ "0" if user.admin else "1" }}">
<button class="btn btn-sm {{ "btn-danger" if user.admin else "btn-success" }}">
<i class="far {{ "fa-check-square" if user.admin else "fa-square" }}"></i>
{{ "Remove admin" if user.admin else "Set admin" }}
</button>
<input type="hidden" name="dir_id" value="{{ dir_id }}">
</form>
</div>
</div>
<hr>
<table class="info-table table-hover table-striped">
<thead>
<tr>
<th>Directory</th>
<th>Search access</th>
<th>Access</th>
</tr>
</thead>
<tbody>
{% for user in users %}
{% for dir_id in directories %}
<tr>
<td>{{ user }}</td>
<td><i class="far {{ "fa-check-square" if users[user].admin else "fa-square" }}"></i></td>
<td><a href="/user/{{ user }}" class="btn btn-primary">Manage</a></td>
<td>{{ directories[dir_id].name }}</td>
<td><form action="/user/{{ user.username }}/set_access" style="display: inline;margin-left: 6px;">
<input type="hidden" name="access" value="{{ "0" if dir_id in user.readable_directories else "1" }}">
<button class="btn btn-sm {{ "btn-danger" if dir_id in user.readable_directories else "btn-success" }}">
<i class="far {{ "fa-check-square" if dir_id in user.readable_directories else "fa-square" }}"></i>
{{ "Disable" if dir_id in user.readable_directories else "Enable" }}
</button>
<input type="hidden" name="dir_id" value="{{ dir_id }}">
</form></td>
</tr>
{% endfor %}
</tbody>
</table>
<hr>
<button class="btn btn-danger" onclick="userDelete()">Delete account</button>
</div>
</div>
<script>
function userDelete() {
if (confirm("Are you sure?")) {
window.location = "/user/{{ user.username }}/del"
}
}
</script>
</div>
{% endblock body %}

7
test/test_MimeGuesser.py
View File

@ -12,7 +12,7 @@ class MimeGuesserTest(TestCase):
guesser = ContentMimeGuesser()
self.assertEqual("text/x-shellscript", guesser.guess_mime(dir_name + "/test_folder/test_utf8.sh"))
self.assertEqual("text/plain", guesser.guess_mime(dir_name + "/test_folder/more_books.json"))
self.assertTrue(guesser.guess_mime(dir_name + "/test_folder/more_books.json") in ["application/json", "text/plain"])
self.assertEqual("application/java-archive", guesser.guess_mime(dir_name + "/test_folder/post.jar"))
self.assertEqual("image/jpeg", guesser.guess_mime(dir_name + "/test_folder/sample_1.jpg"))
@ -20,7 +20,8 @@ class MimeGuesserTest(TestCase):
guesser = ExtensionMimeGuesser()
self.assertEqual("text/x-sh", guesser.guess_mime(dir_name + "/test_folder/test_utf8.sh"))
self.assertTrue(guesser.guess_mime(dir_name + "/test_folder/test_utf8.sh") in ["text/x-sh", "application/x-sh"])
self.assertEqual("application/json", guesser.guess_mime(dir_name + "/test_folder/more_books.json"))
self.assertEqual("application/java-archive", guesser.guess_mime(dir_name + "/test_folder/post.jar"))
self.assertTrue(guesser.guess_mime(dir_name + "/test_folder/post.jar")
in ["application/java-archive", "application/x-java-archive"])
self.assertEqual("image/jpeg", guesser.guess_mime(dir_name + "/test_folder/sample_1.jpg"))