From edede200f4f447003b3171354599a34468f4ba21 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 12 Aug 2018 14:58:27 -0400 Subject: [PATCH] Decresed number of indexed documents per second --- search/search.py | 4 +-- test/files/lighttpd_table.html | 47 ++++++++++++++++++++++++++++++++++ test/files/nginx_pre.html | 11 ++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 test/files/lighttpd_table.html create mode 100644 test/files/nginx_pre.html diff --git a/search/search.py b/search/search.py index 6465de9..fe7b85d 100644 --- a/search/search.py +++ b/search/search.py @@ -125,8 +125,8 @@ class ElasticSearchEngine(SearchEngine): def import_json(self, in_lines, website_id: int): - import_every = 1000 - cooldown_time = 1 + import_every = 400 + cooldown_time = 0.6 docs = [] diff --git a/test/files/lighttpd_table.html b/test/files/lighttpd_table.html new file mode 100644 index 0000000..fe58d9f --- /dev/null +++ b/test/files/lighttpd_table.html @@ -0,0 +1,47 @@ + + + + + Index of /gentoo/releases/ + + + +

Index of /gentoo/releases/

+
+ + + + + + + + + + + + + + + + + + + +
NameLast ModifiedSizeType
Parent Directory/ -  Directory
alpha/2009-Aug-09 03:47:09-  Directory
amd64/2017-Feb-09 18:50:44-  Directory
arm/2014-Apr-29 13:42:06-  Directory
hppa/2014-Apr-29 13:42:12-  Directory
ia64/2009-Aug-09 03:47:09-  Directory
mips/2011-Apr-28 23:38:14-  Directory
ppc/2014-Apr-29 13:41:00-  Directory
s390/2014-Apr-29 13:41:06-  Directory
sh/2014-Apr-29 13:41:16-  Directory
snapshots/2009-Apr-16 05:08:17-  Directory
sparc/2009-Aug-09 03:47:09-  Directory
x86/2016-Jul-04 21:14:19-  Directory
README2014-Jun-22 05:18:430.1Kapplication/octet-stream
verify-digests.sh2016-Jun-10 02:40:334.5Kapplication/octet-stream
+
+
lighttpd/1.4.29
+ + diff --git a/test/files/nginx_pre.html b/test/files/nginx_pre.html new file mode 100644 index 0000000..5bbd35e --- /dev/null +++ b/test/files/nginx_pre.html @@ -0,0 +1,11 @@ + +Index of /test/To process/Android nak newer/ + +

Index of /test/To process/Android nak newer/

../
+DCIM/                                              31-Jul-2018 00:26                   -
+Pictures/                                          31-Jul-2018 00:26                   -
+1529682937580.webm                                 25-Jun-2018 03:58             3768511
+1529716051300.webm                                 25-Jun-2018 04:01             3181867
+1529725898345.webm                                 25-Jun-2018 04:05             4138908
+
+