Basic searching

crawler.py

@@ -132,8 +132,9 @@ class TaskManager:
 
     def execute_thumbnails(self, directory: Directory, total_files: Value, counter: Value, done: Value):
 
-        dest_path = os.path.join("thumbnails", str(directory.id))
+        dest_path = os.path.join("static/thumbnails", str(directory.id))
-        shutil.rmtree(dest_path)
+        if os.path.exists(dest_path):
+            shutil.rmtree(dest_path)
 
         docs = list(Search("changeme").get_all_documents(directory.id))
 

run.py

@@ -1,4 +1,4 @@
-from flask import Flask, render_template, request, redirect, flash, session
+from flask import Flask, render_template, request, redirect, flash, session, abort, send_file
 from storage import Directory, Option, Task
 from storage import LocalStorage, DuplicateDirectoryException
 from crawler import RunningTask, TaskManager
@@ -28,9 +28,60 @@ def get_dir_size(path):
     return size
 
 
+@app.route("/document/<doc_id>")
+def document(doc_id):
+
+    doc = search.get_doc(doc_id)["_source"]
+    directory = storage.dirs()[doc["directory"]]
+
+    del doc["directory"]
+
+    return render_template("document.html", doc=doc, directory=directory, doc_id=doc_id)
+
+
+@app.route("/file/<doc_id>")
+def file(doc_id):
+
+    doc = search.get_doc(doc_id)["_source"]
+    directory = storage.dirs()[doc["directory"]]
+
+    full_path = os.path.join(directory.path, doc["path"], doc["name"])
+
+    return send_file(full_path)
+
+
+@app.route("/thumb/<int:dir_id>/<doc_id>")
+def thumb(dir_id, doc_id):
+
+    if dir_id in storage.dirs():
+
+        return app.send_static_file(os.path.join("thumbnails/", str(dir_id), doc_id))
+
+    else:
+        abort(404)
+
+
 @app.route("/")
-def tmp_route():
+def search_page():
-    return "huh"
+    return render_template("search.html")
+
+
+@app.route("/search")
+def search_route():
+
+    page = search.search()
+
+    return json.dumps(page)
+
+
+@app.route("/scroll")
+def scroll_route():
+
+    scroll_id = request.args.get("scroll_id")
+
+    page = search.scroll(scroll_id)
+
+    return json.dumps(page)
 
 
 @app.route("/directory")

search.py

@@ -16,6 +16,8 @@ class Search:
         except:
             print("elasticsearch is not running")
 
+        self.search_iterator = None
+
     def get_all_documents(self, dir_id: int):
 
         return helpers.scan(client=self.es,
@@ -47,3 +49,19 @@ class Search:
                 return int(parsed_info["indices"][self.index_name]["primaries"]["indexing"]["index_total"])
         except:
             return 0
+
+    def search(self):
+        page = self.es.search(body={"query": {"term": {"directory": 1}}, "size": 30},
+                              index=self.index_name, scroll="3m")
+
+        return page
+
+    def scroll(self, scroll_id):
+
+        page = self.es.scroll(scroll_id=scroll_id, scroll="3m")
+
+        return page
+
+    def get_doc(self, doc_id):
+
+        return self.es.get(index=self.index_name, id=doc_id, doc_type="file")

templates/document.html

@@ -0,0 +1,55 @@
+{% extends "layout.html" %}
+
+{% block body %}
+
+    <link rel="shortcut icon" href="/thumb/{{ directory.id }}/{{ doc_id }}" type="image/jpeg">
+
+    <div class="container">
+
+        <div class="card">
+
+            <div class="card-header">{{ doc.name }}</div>
+            <div class="card-body">
+                <h3>Document properties</h3>
+
+                <table class="info-table table-hover table-striped">
+                    <tbody>
+
+                    {% for key in doc %}
+                        <tr>
+                            <th>{{ key }}</th>
+                            <td><pre>{{ doc[key] }}</pre></td>
+                        </tr>
+                    {% endfor %}
+                    </tbody>
+                </table>
+
+                <hr>
+
+                <h3>Raw json</h3>
+                <textarea class="form-control" style="min-height: 100px" readonly>{{ doc | tojson }}</textarea>
+
+                <hr>
+
+                <h3><a href="/directory/{{ directory.id }}">Directory</a></h3>
+                <table class="info-table table-hover table-striped">
+                    <tbody>
+
+                    <tr>
+                        <th>Path</th>
+                        <td><pre>{{ directory.path }}</pre></td>
+                    </tr>
+                    <tr>
+                        <th>Name</th>
+                        <td><pre>{{ directory.name }}</pre></td>
+                    </tr>
+                    </tbody>
+                </table>
+
+            </div>
+        </div>
+
+    </div>
+
+
+{% endblock body %}

templates/layout.html

@@ -41,7 +41,7 @@
 <body>
 
 <nav class="navbar navbar-expand-lg navbar-light" style="background: #F7F7F7; border-bottom: solid 1px #dfdfdf;">
-    <a class="navbar-brand" href="/search"><i class="fa fa-search"></i> Search</a>
+    <a class="navbar-brand" href="/"><i class="fa fa-search"></i> Search</a>
     <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
         <span class="navbar-toggler-icon"></span>
     </button>

templates/search.html

@@ -0,0 +1,364 @@
+{% extends "layout.html" %}
+
+{% set active_page = "search" %}
+
+{% block title %}Search{% endblock title %}
+
+{% block body %}
+
+    <style>
+        .document {
+            padding: 0.5rem;
+        }
+
+        .document:hover {
+            text-decoration: underline;
+        }
+
+        .card-columns {
+            column-count: 4;
+        }
+
+        .badge-video {
+            color: #ffffff;
+            background-color: #F27761;
+        }
+
+        .badge-image {
+            color: #ffffff;
+            background-color: #AA99C9;
+        }
+
+        .badge-resolution {
+            color: #212529;
+            background-color: #ffc107;
+        }
+
+        .card-img-overlay {
+            pointer-events: none;
+            padding: 0.75rem;
+
+            bottom: 0;
+            top: unset;
+            left: unset;
+            right: unset;
+        }
+
+        .file-title {
+            font-size: 10pt;
+            white-space: nowrap;
+            text-overflow: ellipsis;
+            overflow: hidden;
+        }
+
+        .badge {
+            margin-right: 3px;
+        }
+
+        .fit {
+            width: 100%;
+            height: 100%;
+        }
+
+    </style>
+
+    <div class="container">
+
+        <div class="card">
+            {#            <div class="card-header">An excellent form</div>#}
+            <div class="card-body">
+                <form action=""></form>
+                <input id="searchBar" type="search" class="form-control" placeholder="Search">
+            </div>
+        </div>
+
+        <div id="searchResults">
+
+        </div>
+
+        <script>
+
+            var searchBar = document.getElementById("searchBar");
+            var scroll_id = null;
+            var coolingDown = false;
+            var docCount = 0;
+
+            searchBar.addEventListener("keydown", function () {
+
+                var query = searchBar.value;
+
+                console.log("query: " + query);
+
+                var xhttp = new XMLHttpRequest();
+                xhttp.onreadystatechange = function() {
+                    if (this.readyState === 4 && this.status === 200) {
+
+                        var searchResult = JSON.parse(this.responseText);
+
+                        scroll_id = searchResult["_scroll_id"];
+
+                        var resultContainer = document.createElement("div");
+                        resultContainer.setAttribute("class", "card-columns");
+                        document.getElementById("searchResults").appendChild(resultContainer);
+
+                        for (var i = 0 ; i < searchResult["hits"]["hits"].length; i++) {
+
+                            var hit = searchResult["hits"]["hits"][i];
+                            resultContainer.appendChild(createDocCard(hit));
+                            docCount++;
+                        }
+                    }
+                };
+                xhttp.open("GET", "/search", true);
+                xhttp.send();
+
+            });
+
+            function createDocCard(hit) {
+
+                var doc = document.createElement("div");
+                doc.setAttribute("class", "card");
+
+                var docBody = document.createElement("div");
+                docBody.setAttribute("class", "card-body document");
+
+                var link = document.createElement("a");
+                link.setAttribute("href", "/document/" + hit["_id"]);
+
+                var title = document.createElement("p");
+                title.setAttribute("class", "file-title");
+                title.appendChild(document.createTextNode(hit["_source"]["name"]));
+                title.setAttribute("title", hit["_source"]["name"]);
+                doc.appendChild(title);
+
+                var tagContainer = document.createElement("div");
+                tagContainer.setAttribute("class", "card-text");
+
+                if (hit["_source"].hasOwnProperty("mime")) {
+
+                    var tags = [];
+                    var thumbnail = null;
+                    var thumbnailOverlay = null;
+                    var imgWrapper = document.createElement("div");
+                    imgWrapper.setAttribute("style", "position: relative");
+
+                    var mimeCategory = hit["_source"]["mime"].split("/")[0];
+
+                    //Thumbnail
+                    switch (mimeCategory) {
+
+                        case "video":
+                        case "image":
+                            thumbnail = document.createElement("img");
+                            thumbnail.setAttribute("class", "card-img-top");
+                            thumbnail.setAttribute("src", "/thumb/" + hit["_source"]["directory"] + "/" + hit["_id"]);
+
+                            break;
+
+                    }
+
+                    //Thumbnail overlay
+                    switch (mimeCategory) {
+
+                        case "image":
+                            thumbnailOverlay = document.createElement("div");
+                            thumbnailOverlay.setAttribute("class", "card-img-overlay");
+
+                            //Resolution
+                            var resolutionBadge = document.createElement("span");
+                            resolutionBadge.setAttribute("class", "badge badge-resolution");
+                            resolutionBadge.appendChild(document.createTextNode(hit["_source"]["width"] + "x" + hit["_source"]["height"]));
+                            thumbnailOverlay.appendChild(resolutionBadge);
+
+                            var format = hit["_source"]["format"];
+
+                            //Hover
+                            if(format === "GIF") {
+                                thumbnail.addEventListener("mouseover", function () {
+
+                                    var self = this;
+                                    this.mouseStayedOver = true;
+
+                                    window.setTimeout(function() {
+                                        if (self.mouseStayedOver) {
+                                            thumbnail.removeEventListener('mouseover', arguments.callee, false);
+
+                                            thumbnail.setAttribute("style", "display: none");
+
+                                            var gifImage = document.createElement("img");
+                                            gifImage.setAttribute("src", "/file/" + hit["_id"]);
+                                            gifImage.setAttribute("class", "fit");
+                                            imgWrapper.appendChild(gifImage);
+
+                                            gifImage.addEventListener("mouseout", function() {
+                                                gifImage.setAttribute("style", "display: none");
+                                                thumbnail.setAttribute("style", "");
+                                            });
+                                            thumbnail.addEventListener("mouseover", function () {
+                                                gifImage.setAttribute("style", "");
+                                                thumbnail.setAttribute("style", "display: none");
+                                            });
+                                        }
+                                    }, 750);
+
+
+                                });
+                                thumbnail.addEventListener("mouseout", function() {
+                                    this.mouseStayedOver = false;
+                                })
+                            }
+
+
+
+                            break;
+                        case "video":
+                            thumbnailOverlay = document.createElement("div");
+                            thumbnailOverlay.setAttribute("class", "card-img-overlay");
+
+                            //Duration
+                            var durationBadge = document.createElement("span");
+                            durationBadge.setAttribute("class", "badge badge-resolution");
+                            durationBadge.appendChild(document.createTextNode(parseFloat(hit["_source"]["duration"]).toFixed(2) + "s"));
+                            thumbnailOverlay.appendChild(durationBadge);
+
+
+                            thumbnail.addEventListener("mouseover", function () {
+
+                                var self = this;
+                                this.mouseStayedOver = true;
+
+                                window.setTimeout(function() {
+                                    if(self.mouseStayedOver) {
+                                        imgWrapper.removeChild(thumbnail);
+                                        imgWrapper.removeChild(thumbnailOverlay);
+
+                                        var video = document.createElement("video");
+                                        var vidSource = document.createElement("source");
+                                        vidSource.setAttribute("src", "/file/" + hit["_id"]);
+                                        vidSource.setAttribute("type", "video/webm");
+                                        video.appendChild(vidSource);
+                                        video.setAttribute("class", "fit");
+                                        imgWrapper.appendChild(video);
+
+                                        //Video hover
+                                        video.addEventListener("mouseover", function() {
+                                            var isPlaying = video.currentTime > 0 && !video.paused && !video.ended && video.readyState > 2;
+
+                                            if (!isPlaying) {
+                                                video.play();
+                                            }
+                                        });
+
+                                        video.addEventListener("mouseout", function() {
+                                            video.currentTime = 0;
+                                            video.pause();
+                                        });
+
+                                        video.addEventListener("dblclick", function() {
+                                            video.webkitRequestFullScreen();
+                                        })
+                                    }
+                                }, 750);
+                            });
+                            thumbnail.addEventListener("mouseout", function() {
+                                this.mouseStayedOver = false;
+                            });
+
+                    }
+
+                    //Tags
+                    switch (mimeCategory) {
+
+                        case "video":
+                            var videoTag = document.createElement("span");
+                            videoTag.setAttribute("class", "badge badge-pill badge-video");
+                            videoTag.appendChild(document.createTextNode("video"));
+                            tags.push(videoTag);
+
+                            var formatTag = document.createElement("span");
+                            formatTag.setAttribute("class", "badge badge-pill badge-secondary");
+                            formatTag.appendChild(document.createTextNode(hit["_source"]["format_long_name"]));
+                            tags.push(formatTag);
+
+                            break;
+                        case "image":
+                            var imgTag = document.createElement("span");
+                            imgTag.setAttribute("class", "badge badge-pill badge-image");
+                            imgTag.appendChild(document.createTextNode("image"));
+                            tags.push(imgTag);
+
+                            formatTag = document.createElement("span");
+                            formatTag.setAttribute("class", "badge badge-pill badge-secondary");
+                            formatTag.appendChild(document.createTextNode(format));
+                            tags.push(formatTag);
+
+                            break;
+                    }
+
+                    for (var i = 0; i < tags.length; i++) {
+                        tagContainer.appendChild(tags[i]);
+                    }
+                    if (thumbnail !== null) {
+                        imgWrapper.appendChild(thumbnail);
+                        doc.appendChild(imgWrapper);
+
+                    }
+                    if (thumbnailOverlay !== null) {
+                        imgWrapper.appendChild(thumbnailOverlay);
+                    }
+                }
+
+                docBody.appendChild(link);
+                doc.appendChild(docBody);
+                link.appendChild(title);
+                link.appendChild(tagContainer);
+
+                return doc;
+            }
+
+            window.addEventListener("scroll", function () {
+
+                if (!coolingDown) {
+                    var threshold = 200;
+
+                    if ((window.innerHeight + window.scrollY) >= document.body.offsetHeight - threshold) {
+                        //load next page
+
+                        var xhttp = new XMLHttpRequest();
+                        xhttp.onreadystatechange = function() {
+                            if (this.readyState === 4 && this.status === 200) {
+
+                                var searchResult = JSON.parse(this.responseText);
+
+                                var pageIndicator = document.createElement("div");
+                                pageIndicator.appendChild(document.createTextNode(docCount + " / " +searchResult["hits"]["total"]));
+                                document.getElementById("searchResults").appendChild(pageIndicator);
+
+                                var resultContainer = document.createElement("div");
+                                resultContainer.setAttribute("class", "card-columns");
+                                document.getElementById("searchResults").appendChild(resultContainer);
+
+                                for (var i = 0 ; i < searchResult["hits"]["hits"].length; i++) {
+
+                                    var hit = searchResult["hits"]["hits"][i];
+                                    resultContainer.appendChild(createDocCard(hit));
+                                    docCount++;
+                                }
+
+                                if(searchResult["hits"]["hits"].length !== 0) {
+                                    coolingDown = false;
+                                }
+                            }
+                        };
+                        xhttp.open("GET", "/scroll?scroll_id=" + scroll_id, true);
+                        xhttp.send();
+                        coolingDown = true;
+                    }
+                }
+            })
+
+        </script>
+    </div>
+
+
+{% endblock body %}