version bump

scripts/build.sh

@@ -8,13 +8,15 @@ git submodule update --init --recursive
   cd sist2-vue/
   npm install
   npm run build
-)
+) &
 
 (
   cd sist2-admin/frontend/
   npm install
   npm run build
-)
+) &
+
+wait
 
 mkdir build
 (

scripts/build_arm64.sh

@@ -8,13 +8,15 @@ git submodule update --init --recursive
   cd sist2-vue/
   npm install
   npm run build
-)
+) &
 
 (
   cd sist2-admin/frontend/
   npm install
   npm run build
-)
+) &
+
+wait
 
 mkdir build
 (

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -7,15 +7,15 @@ class Sist2AdminApi {
     }
 
     getJobs() {
-        return axios.get(`${this.baseUrl}/api/job/`);
+        return axios.get(`${this.baseUrl}/api/job`);
     }
 
     getFrontends() {
-        return axios.get(`${this.baseUrl}/api/frontend/`);
+        return axios.get(`${this.baseUrl}/api/frontend`);
     }
 
     getTasks() {
-        return axios.get(`${this.baseUrl}/api/task/`);
+        return axios.get(`${this.baseUrl}/api/task`);
     }
 
     killTask(taskId) {
@@ -42,7 +42,7 @@ class Sist2AdminApi {
     }
 
     getSearchBackends() {
-        return axios.get(`${this.baseUrl}/api/search_backend/`);
+        return axios.get(`${this.baseUrl}/api/search_backend`);
     }
 
     deleteBackend(name) {
@@ -127,7 +127,7 @@ class Sist2AdminApi {
     }
 
     getSist2AdminInfo() {
-        return axios.get(`${this.baseUrl}/api/`);
+        return axios.get(`${this.baseUrl}/api`);
     }
 
     getLogsToDelete(jobName, n) {

sist2-vue/package.json

@@ -13,7 +13,7 @@
     "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "d3": "^7.8.4",
+    "d3": "^5.6.1",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",

sist2-vue/src/components/D3Treemap.vue

@@ -1,10 +1,10 @@
 <template>
-  <div>
-    <b-btn style="float:right;margin-bottom: 10px" @click="downloadTreemap()" variant="primary">
-      {{ $t("download") }}
-    </b-btn>
-    <svg id="treemap"></svg>
-  </div>
+    <div>
+        <b-btn style="float:right;margin-bottom: 10px" @click="downloadTreemap()" variant="primary">
+            {{ $t("download") }}
+        </b-btn>
+        <svg id="treemap"></svg>
+    </div>
 </template>
 
 <script>
@@ -16,252 +16,252 @@ import domtoimage from "dom-to-image";
 
 
 const TILING_MODES = {
-  "squarify": d3.treemapSquarify,
-  "binary": d3.treemapBinary,
-  "sliceDice": d3.treemapSliceDice,
-  "slice": d3.treemapSlice,
-  "dice": d3.treemapDice,
+    "squarify": d3.treemapSquarify,
+    "binary": d3.treemapBinary,
+    "sliceDice": d3.treemapSliceDice,
+    "slice": d3.treemapSlice,
+    "dice": d3.treemapDice,
 };
 
 const COLORS = {
-  "PuBuGn": d3.interpolatePuBuGn,
-  "PuRd": d3.interpolatePuRd,
-  "PuBu": d3.interpolatePuBu,
-  "YlOrBr": d3.interpolateYlOrBr,
-  "YlOrRd": d3.interpolateYlOrRd,
-  "YlGn": d3.interpolateYlGn,
-  "YlGnBu": d3.interpolateYlGnBu,
-  "Plasma": d3.interpolatePlasma,
-  "Magma": d3.interpolateMagma,
-  "Inferno": d3.interpolateInferno,
-  "Viridis": d3.interpolateViridis,
-  "Turbo": d3.interpolateTurbo,
+    "PuBuGn": d3.interpolatePuBuGn,
+    "PuRd": d3.interpolatePuRd,
+    "PuBu": d3.interpolatePuBu,
+    "YlOrBr": d3.interpolateYlOrBr,
+    "YlOrRd": d3.interpolateYlOrRd,
+    "YlGn": d3.interpolateYlGn,
+    "YlGnBu": d3.interpolateYlGnBu,
+    "Plasma": d3.interpolatePlasma,
+    "Magma": d3.interpolateMagma,
+    "Inferno": d3.interpolateInferno,
+    "Viridis": d3.interpolateViridis,
+    "Turbo": d3.interpolateTurbo,
 };
 
 const SIZES = {
-  "small": [800, 600],
-  "medium": [1300, 750],
-  "large": [1900, 900],
-  "x-large": [2800, 1700],
-  "xx-large": [3600, 2000],
+    "small": [800, 600],
+    "medium": [1300, 750],
+    "large": [1900, 900],
+    "x-large": [2800, 1700],
+    "xx-large": [3600, 2000],
 };
 
 
 const uids = {};
 
 function uid(name) {
-  let id = uids[name] || 0;
-  uids[name] = id + 1;
-  return name + id;
+    let id = uids[name] || 0;
+    uids[name] = id + 1;
+    return name + id;
 }
 
 function cascade(root, offset) {
-  const x = new Map;
-  const y = new Map;
-  return root.eachAfter(d => {
-    if (d.children && d.children.length !== 0) {
-      x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
-      y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
-    } else {
-      x.set(d, 0);
-      y.set(d, 0);
-    }
-  }).eachBefore(d => {
-    d.x1 -= 2 * offset * x.get(d);
-    d.y1 -= 2 * offset * y.get(d);
-  });
+    const x = new Map;
+    const y = new Map;
+    return root.eachAfter(d => {
+        if (d.children && d.children.length !== 0) {
+            x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
+            y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
+        } else {
+            x.set(d, 0);
+            y.set(d, 0);
+        }
+    }).eachBefore(d => {
+        d.x1 -= 2 * offset * x.get(d);
+        d.y1 -= 2 * offset * y.get(d);
+    });
 }
 
 function cascadeTreemap(data, svg, width, height, tilingMode, treemapColor) {
-  const root = cascade(
-      d3.treemap()
-          .size([width, height])
-          .tile(TILING_MODES[tilingMode])
-          .paddingOuter(3)
-          .paddingTop(16)
-          .paddingInner(1)
-          .round(true)(
-              d3.hierarchy(data)
-                  .sum(d => d.value)
-                  .sort((a, b) => b.value - a.value)
-          ),
-      3 // treemap.paddingOuter
-  );
+    const root = cascade(
+        d3.treemap()
+            .size([width, height])
+            .tile(TILING_MODES[tilingMode])
+            .paddingOuter(3)
+            .paddingTop(16)
+            .paddingInner(1)
+            .round(true)(
+                d3.hierarchy(data)
+                    .sum(d => d.value)
+                    .sort((a, b) => b.value - a.value)
+            ),
+        3 // treemap.paddingOuter
+    );
 
-  const maxDepth = Math.max(...root.descendants().map(d => d.depth));
-  const color = d3.scaleSequential([maxDepth, -1], COLORS[treemapColor]);
+    const maxDepth = Math.max(...root.descendants().map(d => d.depth));
+    const color = d3.scaleSequential([maxDepth, -1], COLORS[treemapColor]);
 
-  svg.append("filter")
-      .attr("id", "shadow")
-      .append("feDropShadow")
-      .attr("flood-opacity", 0.3)
-      .attr("dx", 0)
-      .attr("stdDeviation", 3);
+    svg.append("filter")
+        .attr("id", "shadow")
+        .append("feDropShadow")
+        .attr("flood-opacity", 0.3)
+        .attr("dx", 0)
+        .attr("stdDeviation", 3);
 
-  const node = svg.selectAll("g")
-      .data(
-          d3.nest()
-              .key(d => d.depth).sortKeys(d3.ascending)
-              .entries(root.descendants())
-      )
-      .join("g")
-      .attr("filter", "url(#shadow)")
-      .selectAll("g")
-      .data(d => d.values)
-      .join("g")
-      .attr("transform", d => `translate(${d.x0},${d.y0})`);
+    const node = svg.selectAll("g")
+        .data(
+            d3.nest()
+                .key(d => d.depth).sortKeys(d3.ascending)
+                .entries(root.descendants())
+        )
+        .join("g")
+        .attr("filter", "url(#shadow)")
+        .selectAll("g")
+        .data(d => d.values)
+        .join("g")
+        .attr("transform", d => `translate(${d.x0},${d.y0})`);
 
-  node.append("title")
-      .text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+    node.append("title")
+        .text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
 
-  node.append("rect")
-      .attr("id", d => (d.nodeUid = uid("node")))
-      .attr("fill", d => color(d.depth))
-      .attr("width", d => d.x1 - d.x0)
-      .attr("height", d => d.y1 - d.y0);
+    node.append("rect")
+        .attr("id", d => (d.nodeUid = uid("node")))
+        .attr("fill", d => color(d.depth))
+        .attr("width", d => d.x1 - d.x0)
+        .attr("height", d => d.y1 - d.y0);
 
-  node.append("clipPath")
-      .attr("id", d => (d.clipUid = uid("clip")))
-      .append("use")
-      .attr("href", d => `#${d.nodeUid}`);
+    node.append("clipPath")
+        .attr("id", d => (d.clipUid = uid("clip")))
+        .append("use")
+        .attr("href", d => `#${d.nodeUid}`);
 
-  node.append("text")
-      .attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
-      .attr("clip-path", d => `url(#${d.clipUid})`)
-      .selectAll("tspan")
-      .data(d => [d.data.name, humanFileSize(d.value)])
-      .join("tspan")
-      .text(d => d);
+    node.append("text")
+        .attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
+        .attr("clip-path", d => `url(#${d.clipUid})`)
+        .selectAll("tspan")
+        .data(d => [d.data.name, humanFileSize(d.value)])
+        .join("tspan")
+        .text(d => d);
 
-  node.filter(d => d.children).selectAll("tspan")
-      .attr("dx", 3)
-      .attr("y", 13);
+    node.filter(d => d.children).selectAll("tspan")
+        .attr("dx", 3)
+        .attr("y", 13);
 
-  node.filter(d => !d.children).selectAll("tspan")
-      .attr("x", 3)
-      .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`);
+    node.filter(d => !d.children).selectAll("tspan")
+        .attr("x", 3)
+        .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`);
 }
 
 function flatTreemap(data, svg, width, height, groupingDepth, tilingMode, fillOpacity) {
-  const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
+    const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
 
-  const root = d3.treemap()
-      .tile(TILING_MODES[tilingMode])
-      .size([width, height])
-      .padding(1)
-      .round(true)(
-          d3.hierarchy(data)
-              .sum(d => d.value)
-              .sort((a, b) => b.value - a.value)
-      );
+    const root = d3.treemap()
+        .tile(TILING_MODES[tilingMode])
+        .size([width, height])
+        .padding(1)
+        .round(true)(
+            d3.hierarchy(data)
+                .sum(d => d.value)
+                .sort((a, b) => b.value - a.value)
+        );
 
-  const leaf = svg.selectAll("g")
-      .data(root.leaves())
-      .join("g")
-      .attr("transform", d => `translate(${d.x0},${d.y0})`);
+    const leaf = svg.selectAll("g")
+        .data(root.leaves())
+        .join("g")
+        .attr("transform", d => `translate(${d.x0},${d.y0})`);
 
-  leaf.append("title")
-      .text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+    leaf.append("title")
+        .text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
 
-  leaf.append("rect")
-      .attr("id", d => (d.leafUid = uid("leaf")))
-      .attr("fill", d => {
-        while (d.depth > groupingDepth) d = d.parent;
-        return ordinalColor(d.data.name);
-      })
-      .attr("fill-opacity", fillOpacity)
-      .attr("width", d => d.x1 - d.x0)
-      .attr("height", d => d.y1 - d.y0);
+    leaf.append("rect")
+        .attr("id", d => (d.leafUid = uid("leaf")))
+        .attr("fill", d => {
+            while (d.depth > groupingDepth) d = d.parent;
+            return ordinalColor(d.data.name);
+        })
+        .attr("fill-opacity", fillOpacity)
+        .attr("width", d => d.x1 - d.x0)
+        .attr("height", d => d.y1 - d.y0);
 
-  leaf.append("clipPath")
-      .attr("id", d => (d.clipUid = uid("clip")))
-      .append("use")
-      .attr("href", d => `#${d.leafUid}`);
+    leaf.append("clipPath")
+        .attr("id", d => (d.clipUid = uid("clip")))
+        .append("use")
+        .attr("href", d => `#${d.leafUid}`);
 
-  leaf.append("text")
-      .attr("clip-path", d => `url(#${d.clipUid})`)
-      .selectAll("tspan")
-      .data(d => {
-        if (d.data.name === ".") {
-          d = d.parent;
-        }
-        return [d.data.name, humanFileSize(d.value)]
-      })
-      .join("tspan")
-      .attr("x", 2)
-      .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`)
-      .text(d => d);
+    leaf.append("text")
+        .attr("clip-path", d => `url(#${d.clipUid})`)
+        .selectAll("tspan")
+        .data(d => {
+            if (d.data.name === ".") {
+                d = d.parent;
+            }
+            return [d.data.name, humanFileSize(d.value)]
+        })
+        .join("tspan")
+        .attr("x", 2)
+        .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`)
+        .text(d => d);
 }
 
 function exportTreemap(indexName, width, height) {
-  domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
-      .then(function (blob) {
-        let a = document.createElement("a");
-        let url = URL.createObjectURL(blob);
+    domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
+        .then(function (blob) {
+            let a = document.createElement("a");
+            let url = URL.createObjectURL(blob);
 
-        a.href = url;
-        a.download = `${indexName}_treemap.png`;
-        document.body.appendChild(a);
-        a.click();
-        setTimeout(function () {
-          document.body.removeChild(a);
-          window.URL.revokeObjectURL(url);
-        }, 0);
-      });
+            a.href = url;
+            a.download = `${indexName}_treemap.png`;
+            document.body.appendChild(a);
+            a.click();
+            setTimeout(function () {
+                document.body.removeChild(a);
+                window.URL.revokeObjectURL(url);
+            }, 0);
+        });
 }
 
 export default {
-  name: "D3Treemap",
-  props: ["indexId"],
-  watch: {
-    indexId: function () {
-      this.update(this.indexId);
-    }
-  },
-  mounted() {
-    this.update(this.indexId);
-  },
-  methods: {
-    update(indexId) {
-      const width = SIZES[this.$store.state.optTreemapSize][0];
-      const height = SIZES[this.$store.state.optTreemapSize][1];
-      const tilingMode = this.$store.state.optTreemapTiling;
-      const groupingDepth = this.$store.state.optTreemapColorGroupingDepth;
-      const treemapColor = this.$store.state.optTreemapColor;
-      const treemapType = this.$store.state.optTreemapType;
-
-      const treemapSvg = d3.select("#treemap");
-
-      treemapSvg.selectAll("*").remove();
-      treemapSvg.attr("viewBox", [0, 0, width, height])
-          .attr("xmlns", "http://www.w3.org/2000/svg")
-          .attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
-          .attr("version", "1.1")
-          .style("overflow", "visible")
-          .style("font", "10px sans-serif");
-
-      d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
-        tabularData.forEach(row => {
-          row.taxonomy = row.path.split("/");
-          row.size = Number(row.size);
-        });
-
-        if (treemapType === "cascaded") {
-          const data = burrow(tabularData, false);
-          cascadeTreemap(data, treemapSvg, width, height, tilingMode, treemapColor);
-        } else {
-          const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
-          const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
-          flatTreemap(data, treemapSvg, width, height, groupingDepth, tilingMode, fillOpacity);
+    name: "D3Treemap",
+    props: ["indexId"],
+    watch: {
+        indexId: function () {
+            this.update(this.indexId);
         }
-      });
     },
-    downloadTreemap() {
-      const width = SIZES[this.$store.state.optTreemapSize][0];
-      const height = SIZES[this.$store.state.optTreemapSize][1];
+    mounted() {
+        this.update(this.indexId);
+    },
+    methods: {
+        update(indexId) {
+            const width = SIZES[this.$store.state.optTreemapSize][0];
+            const height = SIZES[this.$store.state.optTreemapSize][1];
+            const tilingMode = this.$store.state.optTreemapTiling;
+            const groupingDepth = this.$store.state.optTreemapColorGroupingDepth;
+            const treemapColor = this.$store.state.optTreemapColor;
+            const treemapType = this.$store.state.optTreemapType;
 
-      exportTreemap(this.indexId, width, height);
+            const treemapSvg = d3.select("#treemap");
+
+            treemapSvg.selectAll("*").remove();
+            treemapSvg.attr("viewBox", [0, 0, width, height])
+                .attr("xmlns", "http://www.w3.org/2000/svg")
+                .attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
+                .attr("version", "1.1")
+                .style("overflow", "visible")
+                .style("font", "10px sans-serif");
+
+            d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
+                tabularData.forEach(row => {
+                    row.taxonomy = row.path.split("/");
+                    row.size = Number(row.size);
+                });
+
+                if (treemapType === "cascaded") {
+                    const data = burrow(tabularData, false);
+                    cascadeTreemap(data, treemapSvg, width, height, tilingMode, treemapColor);
+                } else {
+                    const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
+                    const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
+                    flatTreemap(data, treemapSvg, width, height, groupingDepth, tilingMode, fillOpacity);
+                }
+            });
+        },
+        downloadTreemap() {
+            const width = SIZES[this.$store.state.optTreemapSize][0];
+            const height = SIZES[this.$store.state.optTreemapSize][1];
+
+            exportTreemap(this.indexId, width, height);
+        }
     }
-  }
 }
 </script>

sist2-vue/src/views/StatsPage.vue

@@ -57,7 +57,9 @@ export default {
                 };
             })
         },
-        indices: () => this.$store.state.indices
+        indices() {
+            return this.$store.state.indices;
+        }
     }
 }
 </script>

src/cli.c

@@ -83,6 +83,10 @@ void exec_args_destroy(exec_args_t *args) {
     free(args);
 }
 
+void sqlite_index_args_destroy(sqlite_index_args_t *args) {
+    // TODO
+}
+
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");

src/cli.h

@@ -134,5 +134,7 @@ void exec_args_destroy(exec_args_t *args);
 
 int exec_args_validate(exec_args_t *args, int argc, const char **argv);
 
+void sqlite_index_args_destroy(sqlite_index_args_t *args);
+
 
 #endif

src/database/database_fts.c

@@ -447,12 +447,16 @@ database_summary_stats_t database_fts_get_date_range(database_t *db) {
     return stats;
 }
 
-char *get_after_where(char **after, fts_sort_t sort) {
+char *get_after_where(char **after, fts_sort_t sort, int sort_asc) {
     if (after == NULL) {
         return NULL;
     }
 
-    return "(sort_var, doc.ROWID) > (?3, ?4)";
+    if (sort_asc) {
+        return "(sort_var, doc.ROWID) > (?3, ?4)";
+    }
+
+    return "(sort_var, doc.ROWID) < (?3, ?4)";
 }
 
 cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
@@ -469,7 +473,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
     char *index_id_where = index_ids_where_clause(index_ids);
     char *mime_where = mime_types_where_clause(mime_types);
     const char *query_where = match_where(query);
-    const char *after_where = get_after_where(after, sort);
+    const char *after_where = get_after_where(after, sort, sort_asc);
     const char *tags_where = tags_where_clause(tags);
 
     if (!query_where && sort == FTS_SORT_SCORE) {

src/main.c

@@ -68,9 +68,7 @@ void database_scan_begin(scan_args_t *args) {
         desc->version_patch = VersionPatch;
 
         // generate new index id based on timestamp
-        unsigned char index_md5[MD5_DIGEST_LENGTH];
-        MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
-        buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
+        md5_hexdigest(&ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), ScanCtx.index.desc.id);
 
         database_initialize(db);
         database_open(db);
@@ -683,6 +681,7 @@ int main(int argc, const char *argv[]) {
     index_args_destroy(index_args);
     web_args_destroy(web_args);
     exec_args_destroy(exec_args);
+    sqlite_index_args_destroy(sqlite_index_args);
 
     return 0;
 }

src/parsing/fs_util.h

@@ -2,15 +2,18 @@
 #define SIST2_FS_UTIL_H
 
 #include "src/sist.h"
+#include <openssl/evp.h>
 
 #define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));};
 
 static int fs_read(struct vfile *f, void *buf, size_t size) {
     if (f->fd == -1) {
-        SHA1_Init(&f->sha1_ctx);
+        f->sha1_ctx = EVP_MD_CTX_new();
+        EVP_DigestInit_ex(f->sha1_ctx, EVP_sha1(), NULL);
 
         f->fd = open(f->filepath, O_RDONLY);
         if (f->fd == -1) {
+            EVP_MD_CTX_free(f->sha1_ctx);
             return -1;
         }
     }
@@ -19,7 +22,7 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
 
     if (ret != 0 && f->calculate_checksum) {
         f->has_checksum = TRUE;
-        safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret);
+        safe_digest_update(f->sha1_ctx, (unsigned char *) buf, ret);
     }
 
     return ret;
@@ -27,8 +30,11 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
 
 static void fs_close(struct vfile *f) {
     if (f->fd != -1) {
-        SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
         close(f->fd);
+        f->fd = -1;
     }
 }
 

src/sist.h

@@ -51,11 +51,11 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.1.2"
+#define VERSION "3.1.4"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
 static const int VersionMinor = 1;
-static const int VersionPatch = 2;
+static const int VersionPatch = 4;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

src/util.h

@@ -7,6 +7,7 @@
 
 #include "third-party/utf8.h/utf8.h"
 #include "libscan/scan.h"
+#include <openssl/evp.h>
 
 
 char *abspath(const char *path);
@@ -86,13 +87,22 @@ static void buf2hex(const unsigned char *buf, size_t buflen, char *hex_string) {
     *s = '\0';
 }
 
+static void md5_hexdigest(void *data, size_t size, char *output) {
+    EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
+    EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL);
+
+    EVP_DigestUpdate(md_ctx, data, size);
+
+    unsigned char digest[MD5_DIGEST_LENGTH];
+    EVP_DigestFinal_ex(md_ctx, digest, NULL);
+    EVP_MD_CTX_free(md_ctx);
+
+    buf2hex(digest, MD5_DIGEST_LENGTH, output);
+}
 
 __always_inline
 static void generate_doc_id(const char *rel_path, char *doc_id) {
-    unsigned char md[MD5_DIGEST_LENGTH];
-
-    MD5((unsigned char *) rel_path, strlen(rel_path), md);
-    buf2hex(md, sizeof(md), doc_id);
+    md5_hexdigest(rel_path, strlen(rel_path), doc_id);
 }
 
 #define MILLISECOND 1000

third-party/libscan/libscan/arc/arc.c

@@ -22,7 +22,11 @@ int should_parse_filtered_file(const char *filepath) {
 }
 
 void arc_close(struct vfile *f) {
-    SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+    if (f->sha1_ctx != NULL) {
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
+    }
 
     if (f->rewind_buffer != NULL) {
         free(f->rewind_buffer);
@@ -59,7 +63,7 @@ int arc_read(struct vfile *f, void *buf, size_t size) {
     if (bytes_read != 0 && bytes_read <= size && f->calculate_checksum) {
         f->has_checksum = TRUE;
 
-        safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, bytes_read);
+        safe_digest_update(f->sha1_ctx, (unsigned char *) buf, bytes_read);
     }
 
     if (bytes_read != size && archive_errno(f->arc) != 0) {
@@ -237,9 +241,12 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
                     sub_job->ext = (int) strlen(sub_job->filepath);
                 }
 
-                SHA1_Init(&sub_job->vfile.sha1_ctx);
+                sub_job->vfile.sha1_ctx = EVP_MD_CTX_new();
+                EVP_DigestInit(sub_job->vfile.sha1_ctx, EVP_sha1());
 
                 ctx->parse(sub_job);
+
+                sub_job->vfile.close(&sub_job->vfile);
             }
         }
 

third-party/libscan/libscan/arc/arc.h

@@ -35,7 +35,8 @@ static int vfile_open_callback(struct archive *a, void *user_data) {
     arc_data_t *data = (arc_data_t *) user_data;
 
     if (!data->f->is_fs_file) {
-        SHA1_Init(&data->f->sha1_ctx);
+        data->f->sha1_ctx = EVP_MD_CTX_new();
+        EVP_DigestInit(data->f->sha1_ctx, EVP_md5());
     }
 
     return ARCHIVE_OK;
@@ -49,7 +50,7 @@ static long vfile_read_callback(struct archive *a, void *user_data, const void *
 
     if (!data->f->is_fs_file && ret > 0) {
         data->f->has_checksum = TRUE;
-        safe_sha1_update(&data->f->sha1_ctx, (unsigned char*)data->buf, ret);
+        safe_digest_update(data->f->sha1_ctx, (unsigned char *) data->buf, ret);
     }
 
     return ret;
@@ -59,7 +60,9 @@ static int vfile_close_callback(struct archive *a, void *user_data) {
     arc_data_t *data = (arc_data_t *) user_data;
 
     if (!data->f->is_fs_file) {
-        SHA1_Final((unsigned char *) data->f->sha1_digest, &data->f->sha1_ctx);
+        EVP_DigestFinal_ex(data->f->sha1_ctx, data->f->sha1_digest, NULL);
+        EVP_MD_CTX_free(data->f->sha1_ctx);
+        data->f->sha1_ctx = NULL;
     }
 
     return ARCHIVE_OK;

third-party/libscan/libscan/ebook/ebook.c

@@ -230,13 +230,6 @@ static int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
     return 0;
 }
 
-static int ocr_progress(fz_context *fzctx, void *user_data, int progress) {
-    scan_ebook_ctx_t *ctx = user_data;
-    CTX_LOG_INFOF("ebook.c", "OCR PROGRESS=%d", progress);
-
-    return 0;
-}
-
 int read_stext(text_buffer_t *tex, fz_stext_page *stext) {
 
     int count = 0;
@@ -414,7 +407,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi
                                                        page_mediabox, TRUE,
                                                        ctx->tesseract_lang,
                                                        ctx->tesseract_path,
-                                                       ocr_progress, ctx);
+                                                       NULL, NULL);
 
                 fz_var(err);
                 fz_try(fzctx)fz_run_page(fzctx, page, ocr_dev, fz_identity, NULL);

third-party/libscan/libscan/media/media.c

@@ -697,9 +697,10 @@ int memfile_open(vfile_t *f, memfile_t *mem) {
     mem->file = fmemopen(mem->buf, mem->size, "rb");
 
     if (f->calculate_checksum) {
-        SHA1_Init(&f->sha1_ctx);
-        safe_sha1_update(&f->sha1_ctx, mem->buf, mem->size);
-        SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+        safe_digest_update(f->sha1_ctx, mem->buf, mem->size);
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
         f->has_checksum = TRUE;
     }
 

third-party/libscan/libscan/scan.h

@@ -8,6 +8,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <openssl/evp.h>
 #include <openssl/md5.h>
 #include <openssl/sha.h>
 
@@ -146,7 +147,7 @@ typedef struct vfile {
     int mtime;
     size_t st_size;
 
-    SHA_CTX sha1_ctx;
+    EVP_MD_CTX *sha1_ctx;
     unsigned char sha1_digest[SHA1_DIGEST_LENGTH];
 
     void *rewind_buffer;

third-party/libscan/libscan/util.h

@@ -6,6 +6,7 @@
 #include "string.h"
 #include "../third-party/utf8.h/utf8.h"
 #include "macros.h"
+#include <openssl/evp.h>
 
 #define STR_STARTS_WITH_CONSTANT(x, y) (strncmp(y, x, sizeof(y) - 1) == 0)
 
@@ -339,7 +340,7 @@ static void *read_all(vfile_t *f, size_t *size) {
 #define STACK_BUFFER_SIZE (size_t)(4096 * 8)
 
 __always_inline
-static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
+static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) {
     unsigned char stack_buf[STACK_BUFFER_SIZE];
 
     void *sha1_buf;
@@ -351,7 +352,7 @@ static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
     }
 
     memcpy(sha1_buf, buf, size);
-    SHA1_Update(ctx, (const void *) sha1_buf, size);
+    EVP_DigestUpdate(ctx, sha1_buf, size);
 
     if (sha1_buf != stack_buf) {
         free(sha1_buf);