From 91f0c9b1f92786b26fe1e0535c477c1d662ab4b1 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sun, 12 Apr 2020 13:11:55 -0400 Subject: [PATCH] Change hash types --- core.go | 57 ++++++--- db.go | 176 +++++++++++----------------- go.mod | 2 +- go.sum | 4 + hasher/hasher.go | 6 +- models.go | 85 ++++++++++++-- models_easyjson.go | 285 ++++++++++++++++++++++++++++++++++++++++----- web/api.go | 29 ++++- 8 files changed, 470 insertions(+), 174 deletions(-) diff --git a/core.go b/core.go index dd373a6..8ca3ea0 100644 --- a/core.go +++ b/core.go @@ -45,35 +45,62 @@ func Init() { DbInit(Pgdb) } -func ComputeHash(data []byte) (*fastimagehash.MultiHash, error) { - h := &fastimagehash.MultiHash{} +func ComputeHash(data []byte) (*Hashes, error) { + h := &Hashes{} + var code fastimagehash.Code - aHash, code := fastimagehash.AHashMem(data, 12) - if code != fastimagehash.Ok { - return nil, errors.Errorf("aHash error: %d", int(code)) - } - dHash, code := fastimagehash.DHashMem(data, 12) + h.DHash8, code = fastimagehash.DHashMem(data, 8) if code != fastimagehash.Ok { return nil, errors.Errorf("dHash error: %d", int(code)) } - mHash, code := fastimagehash.MHashMem(data, 12) + h.DHash16, code = fastimagehash.DHashMem(data, 16) if code != fastimagehash.Ok { return nil, errors.Errorf("dHash error: %d", int(code)) } - pHash, code := fastimagehash.PHashMem(data, 12, 4) + h.DHash32, code = fastimagehash.DHashMem(data, 32) + if code != fastimagehash.Ok { + return nil, errors.Errorf("dHash error: %d", int(code)) + } + + h.MHash8, code = fastimagehash.MHashMem(data, 8) + if code != fastimagehash.Ok { + return nil, errors.Errorf("mHash error: %d", int(code)) + } + h.MHash16, code = fastimagehash.MHashMem(data, 16) + if code != fastimagehash.Ok { + return nil, errors.Errorf("mHash error: %d", int(code)) + } + h.MHash32, code = fastimagehash.MHashMem(data, 32) + if code != fastimagehash.Ok { + return nil, errors.Errorf("mHash error: %d", int(code)) + } + + h.PHash8, code = fastimagehash.PHashMem(data, 8, 4) if code != fastimagehash.Ok { return nil, errors.Errorf("pHash error: %d", int(code)) } - wHash, code := fastimagehash.WHashMem(data, 8, 0, fastimagehash.Haar) + h.PHash16, code = fastimagehash.PHashMem(data, 16, 4) + if code != fastimagehash.Ok { + return nil, errors.Errorf("pHash error: %d", int(code)) + } + h.PHash32, code = fastimagehash.PHashMem(data, 32, 4) + if code != fastimagehash.Ok { + return nil, errors.Errorf("pHash error: %d", int(code)) + } + + h.WHash8, code = fastimagehash.WHashMem(data, 8, 0, fastimagehash.Haar) + if code != fastimagehash.Ok { + return nil, errors.Errorf("wHash error: %d", int(code)) + } + h.WHash16, code = fastimagehash.WHashMem(data, 16, 0, fastimagehash.Haar) + if code != fastimagehash.Ok { + return nil, errors.Errorf("wHash error: %d", int(code)) + } + h.WHash32, code = fastimagehash.WHashMem(data, 32, 0, fastimagehash.Haar) if code != fastimagehash.Ok { return nil, errors.Errorf("wHash error: %d", int(code)) } - h.AHash = *aHash - h.DHash = *dHash - h.MHash = *mHash - h.PHash = *pHash - h.WHash = *wHash return h, nil } diff --git a/db.go b/db.go index 9847687..fae751a 100644 --- a/db.go +++ b/db.go @@ -6,22 +6,18 @@ import ( "crypto/sha1" "crypto/sha256" "errors" + "fmt" "github.com/jackc/pgx" "github.com/jackc/pgx/pgtype" "github.com/mailru/easyjson" - "github.com/simon987/fastimagehash-go" "go.uber.org/zap" ) -const MaxDistance = 30 +const MaxDistance = 100 const MaxLimit = 1000 type Entry struct { - AHash *fastimagehash.Hash - DHash *fastimagehash.Hash - MHash *fastimagehash.Hash - PHash *fastimagehash.Hash - WHash *fastimagehash.Hash + H *Hashes Size int Sha1 [sha1.Size]byte Md5 [md5.Size]byte @@ -52,26 +48,21 @@ func Store(entry *Entry) { } if !imageExists { - _, err = Pgdb.Exec("INSERT INTO hash_ahash VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.AHash.Bytes) - if err != nil { - Logger.Error("Could not insert ahash", zap.Error(err)) - } - _, err = Pgdb.Exec("INSERT INTO hash_dhash VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.DHash.Bytes) - if err != nil { - Logger.Error("Could not insert dhash", zap.Error(err)) - } - _, err = Pgdb.Exec("INSERT INTO hash_mhash VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.MHash.Bytes) - if err != nil { - Logger.Error("Could not insert mhash", zap.Error(err)) - } - _, err = Pgdb.Exec("INSERT INTO hash_phash VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.PHash.Bytes) - if err != nil { - Logger.Error("Could not insert phash", zap.Error(err)) - } - _, err = Pgdb.Exec("INSERT INTO hash_whash VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.WHash.Bytes) - if err != nil { - Logger.Error("Could not insert whash", zap.Error(err)) - } + _, _ = Pgdb.Exec("INSERT INTO hash_dhash8 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.DHash8.Bytes) + _, _ = Pgdb.Exec("INSERT INTO hash_dhash16 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.DHash16.Bytes) + _, _ = Pgdb.Exec("INSERT INTO hash_dhash32 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.DHash32.Bytes) + + _, _ = Pgdb.Exec("INSERT INTO hash_mhash8 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.MHash8.Bytes) + _, _ = Pgdb.Exec("INSERT INTO hash_mhash16 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.MHash16.Bytes) + _, _ = Pgdb.Exec("INSERT INTO hash_mhash32 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.MHash32.Bytes) + + _, _ = Pgdb.Exec("INSERT INTO hash_phash8 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.PHash8.Bytes) + _, _ = Pgdb.Exec("INSERT INTO hash_phash16 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.PHash16.Bytes) + _, _ = Pgdb.Exec("INSERT INTO hash_phash32 VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.PHash32.Bytes) + + _, _ = Pgdb.Exec("INSERT INTO hash_whash8haar VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.WHash8.Bytes) + _, _ = Pgdb.Exec("INSERT INTO hash_whash16haar VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.WHash16.Bytes) + _, _ = Pgdb.Exec("INSERT INTO hash_whash32haar VALUES ($1, $2) ON CONFLICT DO NOTHING", id, entry.H.WHash32.Bytes) } for _, meta := range entry.Meta { @@ -96,30 +87,36 @@ func Store(entry *Entry) { func isHashValid(hash []byte, hashType HashType) bool { switch hashType { - case AHash12: - if len(hash) != 18 { - return false - } - case DHash12: - if len(hash) != 18 { - return false - } - case MHash12: - if len(hash) != 18 { - return false - } - case PHash12: - if len(hash) != 18 { - return false - } + case DHash8: + fallthrough + case MHash8: + fallthrough + case PHash8: + fallthrough case WHash8Haar: - if len(hash) != 8 { - return false - } + return len(hash) == 8 + + case DHash16: + fallthrough + case MHash16: + fallthrough + case PHash16: + fallthrough + case WHash16Haar: + return len(hash) == 32 + + case DHash32: + fallthrough + case MHash32: + fallthrough + case PHash32: + fallthrough + case WHash32Haar: + return len(hash) == 128 + default: return false } - return true } func FindImagesByHash(ctx context.Context, hash []byte, hashType HashType, distance, limit, offset uint) ([]byte, error) { @@ -143,23 +140,11 @@ func FindImagesByHash(ctx context.Context, hash []byte, hashType HashType, dista defer tx.Commit() var sql string - switch hashType { - case AHash12: - sql = `SELECT image.* FROM image INNER JOIN hash_ahash h on image.id = h.image_id - WHERE hash_is_within_distance18(h.hash, $1, $2) ORDER BY image.id LIMIT $3 OFFSET $4` - case DHash12: - sql = `SELECT image.* FROM image INNER JOIN hash_dhash h on image.id = h.image_id - WHERE hash_is_within_distance18(h.hash, $1, $2) ORDER BY image.id LIMIT $3 OFFSET $4` - case MHash12: - sql = `SELECT image.* FROM image INNER JOIN hash_mhash h on image.id = h.image_id - WHERE hash_is_within_distance18(h.hash, $1, $2) ORDER BY image.id LIMIT $3 OFFSET $4` - case PHash12: - sql = `SELECT image.* FROM image INNER JOIN hash_phash h on image.id = h.image_id - WHERE hash_is_within_distance18(h.hash, $1, $2) ORDER BY image.id LIMIT $3 OFFSET $4` - case WHash8Haar: - sql = `SELECT image.* FROM image INNER JOIN hash_whash h on image.id = h.image_id - WHERE hash_is_within_distance8(h.hash, $1, $2) ORDER BY image.id LIMIT $3 OFFSET $4` - } + sql = fmt.Sprintf( + `SELECT image.* FROM image INNER JOIN hash_%s h on image.id = h.image_id + WHERE hash_is_within_distance%d(h.hash, $1, $2) + ORDER BY image.id LIMIT $3 OFFSET $4`, + hashType, hashType.HashLength()) rows, err := tx.Query(sql, hash, distance, limit, offset) if err != nil { @@ -169,7 +154,7 @@ func FindImagesByHash(ctx context.Context, hash []byte, hashType HashType, dista var images []*Image for rows.Next() { var im Image - err := rows.Scan(&im.id, &im.Size, &im.Sha1, &im.Md5, &im.Sha256, &im.Crc32) + err := rows.Scan(&im.id, &im.Crc32, &im.Size, &im.Sha1, &im.Md5, &im.Sha256) if err != nil { Logger.Error("Error while fetching db image", zap.String("err", err.Error())) return nil, err @@ -178,6 +163,10 @@ func FindImagesByHash(ctx context.Context, hash []byte, hashType HashType, dista images = append(images, &im) } + if images == nil { + return nil, nil + } + batch := tx.BeginBatch() defer batch.Close() for _, im := range images { @@ -222,56 +211,33 @@ func DbInit(pool *pgx.ConnPool) { sql := ` CREATE TABLE IF NOT EXISTS image ( - id BIGSERIAL PRIMARY KEY, - size INT, - sha1 bytea, - md5 bytea, - sha256 bytea, - crc32 bigint + id BIGSERIAL PRIMARY KEY NOT NULL, + crc32 bigint NOT NULL, + size INT NOT NULL, + sha1 bytea NOT NULL, + md5 bytea NOT NULL, + sha256 bytea NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS idx_image_sha1 ON image(sha1); -CREATE INDEX IF NOT EXISTS idx_image_md5 ON image(md5); -CREATE INDEX IF NOT EXISTS idx_image_sha256 ON image(sha256); -CREATE INDEX IF NOT EXISTS idx_image_crc32 ON image(crc32); CREATE TABLE IF NOT EXISTS image_meta ( - id TEXT UNIQUE, - retrieved_at bigint, - meta bytea + id TEXT UNIQUE NOT NULL, + retrieved_at bigint NOT NULL, + meta bytea NOT NULL ); CREATE TABLE IF NOT EXISTS image_has_meta ( - image_id bigint REFERENCES image(id), - url TEXT, - image_meta_id text REFERENCES image_meta(id), + image_id bigint REFERENCES image(id) NOT NULL, + url TEXT NOT NULL, + image_meta_id text REFERENCES image_meta(id) NOT NULL, UNIQUE(image_id, image_meta_id) ); - -CREATE TABLE IF NOT EXISTS hash_phash ( - image_id BIGINT REFERENCES image(id) UNIQUE, - hash bytea -); - -CREATE TABLE IF NOT EXISTS hash_ahash ( - image_id BIGINT REFERENCES image(id) UNIQUE, - hash bytea -); - -CREATE TABLE IF NOT EXISTS hash_dhash ( - image_id BIGINT REFERENCES image(id) UNIQUE, - hash bytea -); - -CREATE TABLE IF NOT EXISTS hash_mhash ( - image_id BIGINT REFERENCES image(id) UNIQUE, - hash bytea -); - -CREATE TABLE IF NOT EXISTS hash_whash ( - image_id BIGINT REFERENCES image(id) UNIQUE, - hash bytea -); ` + for _, hashType := range HashTypes { + sql += fmt.Sprintf(`CREATE TABLE IF NOT EXISTS hash_%s ( + image_id BIGINT REFERENCES image(id) UNIQUE NOT NULL, + hash bytea NOT NULL);`, hashType) + } _, err := pool.Exec(sql) if err != nil { diff --git a/go.mod b/go.mod index cfa046b..c7d157c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/onsi/gomega v1.9.0 // indirect github.com/pkg/errors v0.9.1 github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc // indirect - github.com/simon987/fastimagehash-go v0.0.0-20200411154912-569fe641b1a7 + github.com/simon987/fastimagehash-go v0.0.0-20200412154506-b0e9d9b3a73e github.com/stretchr/testify v1.5.1 // indirect github.com/valyala/fasthttp v1.9.0 go.uber.org/zap v1.14.1 diff --git a/go.sum b/go.sum index df762b9..c728835 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,10 @@ github.com/simon987/fastimagehash-go v0.0.0-20200411005122-1886a7c50720 h1:0VrGo github.com/simon987/fastimagehash-go v0.0.0-20200411005122-1886a7c50720/go.mod h1:MbqNG+6OaprdElEIes1aYF7qmLlaTop4j5X6pgNiaaw= github.com/simon987/fastimagehash-go v0.0.0-20200411154912-569fe641b1a7 h1:4XD2rCg4hJRcCZErDLp8lfMsHw5Zinr5e2t2C18GdzU= github.com/simon987/fastimagehash-go v0.0.0-20200411154912-569fe641b1a7/go.mod h1:fmgaZptm6M5Kn3Ctu/R5p2fncGYPpGi/raZCZUrkRsY= +github.com/simon987/fastimagehash-go v0.0.0-20200412153922-bcfdf954b464 h1:5p3TX77zQoyrAJqrczYLmAo4F8U+dsyHylzZ3MH4sps= +github.com/simon987/fastimagehash-go v0.0.0-20200412153922-bcfdf954b464/go.mod h1:fmgaZptm6M5Kn3Ctu/R5p2fncGYPpGi/raZCZUrkRsY= +github.com/simon987/fastimagehash-go v0.0.0-20200412154506-b0e9d9b3a73e h1:8+cH+kriBBb9OqtKh/wNsr+PvV8e73yNjEly5wAjFQk= +github.com/simon987/fastimagehash-go v0.0.0-20200412154506-b0e9d9b3a73e/go.mod h1:fmgaZptm6M5Kn3Ctu/R5p2fncGYPpGi/raZCZUrkRsY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/hasher/hasher.go b/hasher/hasher.go index 8be99cc..baf5686 100644 --- a/hasher/hasher.go +++ b/hasher/hasher.go @@ -84,11 +84,7 @@ func computeAndStore(rawTask []string) { } Store(&Entry{ - AHash: &h.AHash, - DHash: &h.DHash, - MHash: &h.MHash, - PHash: &h.PHash, - WHash: &h.WHash, + H: h, Size: len(data), Sha256: sha256.Sum256(data), Sha1: sha1.Sum(data), diff --git a/models.go b/models.go index 0e3e471..ecd2bf0 100644 --- a/models.go +++ b/models.go @@ -1,25 +1,86 @@ package imhashdb +import "github.com/simon987/fastimagehash-go" + type HashType string const ( - AHash12 HashType = "ahash:12" - DHash12 HashType = "dhash:12" - MHash12 HashType = "mhash:12" - PHash12 HashType = "phaash:12:4" - WHash8Haar HashType = "whash:8:haar" + DHash8 HashType = "dhash8" + DHash16 HashType = "dhash16" + DHash32 HashType = "dhash32" + + MHash8 HashType = "mhash8" + MHash16 HashType = "mhash16" + MHash32 HashType = "mhash32" + + PHash8 HashType = "phash8" + PHash16 HashType = "phash16" + PHash32 HashType = "phash32" + + WHash8Haar HashType = "whash8haar" + WHash16Haar HashType = "whash16haar" + WHash32Haar HashType = "whash32haar" ) +var HashTypes = []HashType{ + DHash8, DHash16, DHash32, + MHash8, MHash16, MHash32, + PHash8, PHash16, PHash32, + WHash8Haar, WHash16Haar, WHash32Haar, +} + +func (h HashType) HashLength() int { + switch h { + case DHash8: + fallthrough + case MHash8: + fallthrough + case PHash8: + fallthrough + case WHash8Haar: + return 8 + + case DHash16: + fallthrough + case MHash16: + fallthrough + case PHash16: + fallthrough + case WHash16Haar: + return 32 + + case DHash32: + fallthrough + case MHash32: + fallthrough + case PHash32: + fallthrough + case WHash32Haar: + return 128 + default: + panic("Invalid invalid hash") + } +} + type HashReq struct { Data []byte `json:"data"` } +type Hashes struct { + DHash8 *fastimagehash.Hash `json:"dhash8"` + DHash16 *fastimagehash.Hash `json:"dhash16"` + DHash32 *fastimagehash.Hash `json:"dhash32"` -type HashResp struct { - AHash []byte `json:"ahash:12"` - DHash []byte `json:"dhash:12"` - MHash []byte `json:"mhash:12"` - PHash []byte `json:"phash:12:4"` - WHash []byte `json:"whash:18:haar"` + MHash8 *fastimagehash.Hash `json:"mhash8"` + MHash16 *fastimagehash.Hash `json:"mhash16"` + MHash32 *fastimagehash.Hash `json:"mhash32"` + + PHash8 *fastimagehash.Hash `json:"phash8"` + PHash16 *fastimagehash.Hash `json:"phash16"` + PHash32 *fastimagehash.Hash `json:"phash32"` + + WHash8 *fastimagehash.Hash `json:"whash8haar"` + WHash16 *fastimagehash.Hash `json:"whash16haar"` + WHash32 *fastimagehash.Hash `json:"whash32haar"` } type QueryReq struct { @@ -35,7 +96,7 @@ type ImageList struct { } type QueryResp struct { - Err string `json:"err,omitempty"` + Err string `json:"err,omitempty"` } type Meta struct { diff --git a/models_easyjson.go b/models_easyjson.go index bc57ce4..1f5a5a1 100644 --- a/models_easyjson.go +++ b/models_easyjson.go @@ -7,6 +7,7 @@ import ( easyjson "github.com/mailru/easyjson" jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" + fastimagehash_go "github.com/simon987/fastimagehash-go" ) // suppress unused package warning @@ -599,7 +600,7 @@ func (v *Image) UnmarshalJSON(data []byte) error { func (v *Image) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjsonD2b7633eDecodeGithubComSimon987Imhashdb5(l, v) } -func easyjsonD2b7633eDecodeGithubComSimon987Imhashdb6(in *jlexer.Lexer, out *HashResp) { +func easyjsonD2b7633eDecodeGithubComSimon987Imhashdb6(in *jlexer.Lexer, out *Hashes) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -618,40 +619,125 @@ func easyjsonD2b7633eDecodeGithubComSimon987Imhashdb6(in *jlexer.Lexer, out *Has continue } switch key { - case "ahash:12": + case "dhash8": if in.IsNull() { in.Skip() - out.AHash = nil + out.DHash8 = nil } else { - out.AHash = in.Bytes() + if out.DHash8 == nil { + out.DHash8 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.DHash8) } - case "dhash:12": + case "dhash16": if in.IsNull() { in.Skip() - out.DHash = nil + out.DHash16 = nil } else { - out.DHash = in.Bytes() + if out.DHash16 == nil { + out.DHash16 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.DHash16) } - case "mhash:12": + case "dhash32": if in.IsNull() { in.Skip() - out.MHash = nil + out.DHash32 = nil } else { - out.MHash = in.Bytes() + if out.DHash32 == nil { + out.DHash32 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.DHash32) } - case "phash:12:4": + case "mhash8": if in.IsNull() { in.Skip() - out.PHash = nil + out.MHash8 = nil } else { - out.PHash = in.Bytes() + if out.MHash8 == nil { + out.MHash8 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.MHash8) } - case "whash:18:haar": + case "mhash16": if in.IsNull() { in.Skip() - out.WHash = nil + out.MHash16 = nil } else { - out.WHash = in.Bytes() + if out.MHash16 == nil { + out.MHash16 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.MHash16) + } + case "mhash32": + if in.IsNull() { + in.Skip() + out.MHash32 = nil + } else { + if out.MHash32 == nil { + out.MHash32 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.MHash32) + } + case "phash8": + if in.IsNull() { + in.Skip() + out.PHash8 = nil + } else { + if out.PHash8 == nil { + out.PHash8 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.PHash8) + } + case "phash16": + if in.IsNull() { + in.Skip() + out.PHash16 = nil + } else { + if out.PHash16 == nil { + out.PHash16 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.PHash16) + } + case "phash32": + if in.IsNull() { + in.Skip() + out.PHash32 = nil + } else { + if out.PHash32 == nil { + out.PHash32 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.PHash32) + } + case "whash8haar": + if in.IsNull() { + in.Skip() + out.WHash8 = nil + } else { + if out.WHash8 == nil { + out.WHash8 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.WHash8) + } + case "whash16haar": + if in.IsNull() { + in.Skip() + out.WHash16 = nil + } else { + if out.WHash16 == nil { + out.WHash16 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.WHash16) + } + case "whash32haar": + if in.IsNull() { + in.Skip() + out.WHash32 = nil + } else { + if out.WHash32 == nil { + out.WHash32 = new(fastimagehash_go.Hash) + } + easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in, out.WHash32) } default: in.SkipRecursive() @@ -663,61 +749,198 @@ func easyjsonD2b7633eDecodeGithubComSimon987Imhashdb6(in *jlexer.Lexer, out *Has in.Consumed() } } -func easyjsonD2b7633eEncodeGithubComSimon987Imhashdb6(out *jwriter.Writer, in HashResp) { +func easyjsonD2b7633eEncodeGithubComSimon987Imhashdb6(out *jwriter.Writer, in Hashes) { out.RawByte('{') first := true _ = first { - const prefix string = ",\"ahash:12\":" + const prefix string = ",\"dhash8\":" out.RawString(prefix[1:]) - out.Base64Bytes(in.AHash) + if in.DHash8 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.DHash8) + } } { - const prefix string = ",\"dhash:12\":" + const prefix string = ",\"dhash16\":" out.RawString(prefix) - out.Base64Bytes(in.DHash) + if in.DHash16 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.DHash16) + } } { - const prefix string = ",\"mhash:12\":" + const prefix string = ",\"dhash32\":" out.RawString(prefix) - out.Base64Bytes(in.MHash) + if in.DHash32 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.DHash32) + } } { - const prefix string = ",\"phash:12:4\":" + const prefix string = ",\"mhash8\":" out.RawString(prefix) - out.Base64Bytes(in.PHash) + if in.MHash8 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.MHash8) + } } { - const prefix string = ",\"whash:18:haar\":" + const prefix string = ",\"mhash16\":" out.RawString(prefix) - out.Base64Bytes(in.WHash) + if in.MHash16 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.MHash16) + } + } + { + const prefix string = ",\"mhash32\":" + out.RawString(prefix) + if in.MHash32 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.MHash32) + } + } + { + const prefix string = ",\"phash8\":" + out.RawString(prefix) + if in.PHash8 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.PHash8) + } + } + { + const prefix string = ",\"phash16\":" + out.RawString(prefix) + if in.PHash16 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.PHash16) + } + } + { + const prefix string = ",\"phash32\":" + out.RawString(prefix) + if in.PHash32 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.PHash32) + } + } + { + const prefix string = ",\"whash8haar\":" + out.RawString(prefix) + if in.WHash8 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.WHash8) + } + } + { + const prefix string = ",\"whash16haar\":" + out.RawString(prefix) + if in.WHash16 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.WHash16) + } + } + { + const prefix string = ",\"whash32haar\":" + out.RawString(prefix) + if in.WHash32 == nil { + out.RawString("null") + } else { + easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out, *in.WHash32) + } } out.RawByte('}') } // MarshalJSON supports json.Marshaler interface -func (v HashResp) MarshalJSON() ([]byte, error) { +func (v Hashes) MarshalJSON() ([]byte, error) { w := jwriter.Writer{} easyjsonD2b7633eEncodeGithubComSimon987Imhashdb6(&w, v) return w.Buffer.BuildBytes(), w.Error } // MarshalEasyJSON supports easyjson.Marshaler interface -func (v HashResp) MarshalEasyJSON(w *jwriter.Writer) { +func (v Hashes) MarshalEasyJSON(w *jwriter.Writer) { easyjsonD2b7633eEncodeGithubComSimon987Imhashdb6(w, v) } // UnmarshalJSON supports json.Unmarshaler interface -func (v *HashResp) UnmarshalJSON(data []byte) error { +func (v *Hashes) UnmarshalJSON(data []byte) error { r := jlexer.Lexer{Data: data} easyjsonD2b7633eDecodeGithubComSimon987Imhashdb6(&r, v) return r.Error() } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *HashResp) UnmarshalEasyJSON(l *jlexer.Lexer) { +func (v *Hashes) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjsonD2b7633eDecodeGithubComSimon987Imhashdb6(l, v) } +func easyjsonD2b7633eDecodeGithubComSimon987FastimagehashGo(in *jlexer.Lexer, out *fastimagehash_go.Hash) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "Size": + out.Size = int(in.Int()) + case "Bytes": + if in.IsNull() { + in.Skip() + out.Bytes = nil + } else { + out.Bytes = in.Bytes() + } + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjsonD2b7633eEncodeGithubComSimon987FastimagehashGo(out *jwriter.Writer, in fastimagehash_go.Hash) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"Size\":" + out.RawString(prefix[1:]) + out.Int(int(in.Size)) + } + { + const prefix string = ",\"Bytes\":" + out.RawString(prefix) + out.Base64Bytes(in.Bytes) + } + out.RawByte('}') +} func easyjsonD2b7633eDecodeGithubComSimon987Imhashdb7(in *jlexer.Lexer, out *HashReq) { isTopLevel := in.IsStart() if in.IsNull() { diff --git a/web/api.go b/web/api.go index 9285417..2ab1312 100644 --- a/web/api.go +++ b/web/api.go @@ -14,6 +14,23 @@ import ( "time" ) +func CORSMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Max-Age", "86400") + c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE, UPDATE") + c.Writer.Header().Set("Access-Control-Allow-Headers", "X-Requested-With, Content-Type, Origin, Authorization, Accept, Client-Security-Token, Accept-Encoding, x-access-token") + c.Writer.Header().Set("Access-Control-Expose-Headers", "Content-Length") + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(200) + } else { + c.Next() + } + } +} + func submitQuery(value string) bool { if Rdb.SIsMember(wipQueue, value).Val() { return false @@ -80,17 +97,18 @@ func hash(c *gin.Context) { return } + if req.Data == nil { + c.JSON(400, gin.H{"err": "Invalid request"}) + return + } + h, err := ComputeHash(req.Data) if err != nil { c.JSON(500, gin.H{"err": "Couldn't compute image hash"}) return } - b, _ := easyjson.Marshal(HashResp{ - AHash: h.AHash.Bytes, DHash: h.DHash.Bytes, - MHash: h.MHash.Bytes, PHash: h.PHash.Bytes, - WHash: h.WHash.Bytes, - }) + b, _ := easyjson.Marshal(h) c.Data(200, gin.MIMEJSON, b) } @@ -110,6 +128,7 @@ func main() { }() r := gin.Default() + r.Use(CORSMiddleware()) r.POST("/api/hash", hash) r.POST("/api/query", query)