mirror of
https://github.com/simon987/nyaa.git
synced 2025-12-14 07:39:03 +00:00
[Schema change] Torrents flags bitflag column to indexed columns (#471)
* convert torrent table flags column from bitflag to independent indexed columns * elasticsearch integration (untested) * improve performance
This commit is contained in:
A nyaa developer
committed by
Arylide
Arylide
parent
c786bd20f8
commit
41a2a32f66
@@ -67,42 +67,6 @@ class DeclarativeHelperBase(object):
|
||||
return cls._table_prefix(cls.__tablename_base__)
|
||||
|
||||
|
||||
class FlagProperty(object):
|
||||
''' This class will act as a wrapper between the given flag and the class's
|
||||
flag collection. '''
|
||||
|
||||
def __init__(self, flag, flags_attr='flags'):
|
||||
self._flag = flag
|
||||
self._flags_attr_name = flags_attr
|
||||
|
||||
def _get_flags(self, instance):
|
||||
return getattr(instance, self._flags_attr_name)
|
||||
|
||||
def _set_flags(self, instance, value):
|
||||
return setattr(instance, self._flags_attr_name, value)
|
||||
|
||||
def __get__(self, instance, owner_class):
|
||||
if instance is None:
|
||||
raise AttributeError()
|
||||
return bool(self._get_flags(instance) & self._flag)
|
||||
|
||||
def __set__(self, instance, value):
|
||||
new_flags = (self._get_flags(instance) & ~self._flag) | (bool(value) and self._flag)
|
||||
self._set_flags(instance, new_flags)
|
||||
|
||||
|
||||
class TorrentFlags(IntEnum):
|
||||
NONE = 0
|
||||
ANONYMOUS = 1
|
||||
HIDDEN = 2
|
||||
TRUSTED = 4
|
||||
REMAKE = 8
|
||||
COMPLETE = 16
|
||||
DELETED = 32
|
||||
BANNED = 64
|
||||
COMMENT_LOCKED = 128
|
||||
|
||||
|
||||
class TorrentBase(DeclarativeHelperBase):
|
||||
__tablename_base__ = 'torrents'
|
||||
|
||||
@@ -116,12 +80,11 @@ class TorrentBase(DeclarativeHelperBase):
|
||||
|
||||
filesize = db.Column(db.BIGINT, default=0, nullable=False, index=True)
|
||||
encoding = db.Column(db.String(length=32), nullable=False)
|
||||
flags = db.Column(db.Integer, default=0, nullable=False, index=True)
|
||||
|
||||
@declarative.declared_attr
|
||||
def uploader_id(cls):
|
||||
# Even though this is same for both tables, declarative requires this
|
||||
return db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
|
||||
return db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True, index=True)
|
||||
|
||||
uploader_ip = db.Column(db.Binary(length=16), default=None, nullable=True)
|
||||
has_torrent = db.Column(db.Boolean, nullable=False, default=False)
|
||||
@@ -132,6 +95,15 @@ class TorrentBase(DeclarativeHelperBase):
|
||||
updated_time = db.Column(db.DateTime(timezone=False), default=datetime.utcnow,
|
||||
onupdate=datetime.utcnow, nullable=False)
|
||||
|
||||
anonymous = db.Column(db.Boolean, nullable=False, default=False, index=True)
|
||||
hidden = db.Column(db.Boolean, nullable=False, default=False, index=True)
|
||||
deleted = db.Column(db.Boolean, nullable=False, default=False, index=True)
|
||||
banned = db.Column(db.Boolean, nullable=False, default=False, index=True)
|
||||
trusted = db.Column(db.Boolean, nullable=False, default=False, index=True)
|
||||
remake = db.Column(db.Boolean, nullable=False, default=False, index=True)
|
||||
complete = db.Column(db.Boolean, nullable=False, default=False, index=True)
|
||||
comment_locked = db.Column(db.Boolean, nullable=False, default=False, index=True)
|
||||
|
||||
@declarative.declared_attr
|
||||
def main_category_id(cls):
|
||||
fk = db.ForeignKey(cls._table_prefix('main_categories.id'))
|
||||
@@ -147,7 +119,9 @@ class TorrentBase(DeclarativeHelperBase):
|
||||
@declarative.declared_attr
|
||||
def __table_args__(cls):
|
||||
return (
|
||||
Index(cls._table_prefix('uploader_flag_idx'), 'uploader_id', 'flags'),
|
||||
Index('ix_' + cls._table_prefix('super'), 'id', 'uploader_id',
|
||||
'main_category_id', 'sub_category_id',
|
||||
'anonymous', 'hidden', 'deleted', 'banned', 'trusted', 'remake', 'complete'),
|
||||
ForeignKeyConstraint(
|
||||
['main_category_id', 'sub_category_id'],
|
||||
[cls._table_prefix('sub_categories.main_category_id'),
|
||||
@@ -251,17 +225,6 @@ class TorrentBase(DeclarativeHelperBase):
|
||||
if self.uploader_ip:
|
||||
return str(ip_address(self.uploader_ip))
|
||||
|
||||
# Flag properties below
|
||||
|
||||
anonymous = FlagProperty(TorrentFlags.ANONYMOUS)
|
||||
hidden = FlagProperty(TorrentFlags.HIDDEN)
|
||||
deleted = FlagProperty(TorrentFlags.DELETED)
|
||||
banned = FlagProperty(TorrentFlags.BANNED)
|
||||
trusted = FlagProperty(TorrentFlags.TRUSTED)
|
||||
remake = FlagProperty(TorrentFlags.REMAKE)
|
||||
complete = FlagProperty(TorrentFlags.COMPLETE)
|
||||
comment_locked = FlagProperty(TorrentFlags.COMMENT_LOCKED)
|
||||
|
||||
# Class methods
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -8,6 +8,7 @@ import sqlalchemy
|
||||
import sqlalchemy_fulltext.modes as FullTextMode
|
||||
from elasticsearch import Elasticsearch
|
||||
from elasticsearch_dsl import Q, Search
|
||||
from sqlalchemy.sql.expression import false
|
||||
from sqlalchemy_fulltext import FullTextSearch
|
||||
|
||||
from nyaa import models
|
||||
@@ -294,9 +295,9 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
|
||||
|
||||
filter_keys = {
|
||||
'0': None,
|
||||
'1': (models.TorrentFlags.REMAKE, False),
|
||||
'2': (models.TorrentFlags.TRUSTED, True),
|
||||
'3': (models.TorrentFlags.COMPLETE, True)
|
||||
'1': (models.Torrent.remake, False),
|
||||
'2': (models.Torrent.trusted, True),
|
||||
'3': (models.Torrent.complete, True)
|
||||
}
|
||||
|
||||
sentinel = object()
|
||||
@@ -356,8 +357,7 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
|
||||
|
||||
if not admin:
|
||||
# Hide all DELETED torrents if regular user
|
||||
qpc.filter(models.Torrent.flags.op('&')(
|
||||
int(models.TorrentFlags.DELETED)).is_(False))
|
||||
qpc.filter(models.Torrent.deleted == false())
|
||||
# If logged in user is not the same as the user being viewed,
|
||||
# show only torrents that aren't hidden or anonymous
|
||||
#
|
||||
@@ -367,24 +367,21 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
|
||||
# On RSS pages in user view,
|
||||
# show only torrents that aren't hidden or anonymous no matter what
|
||||
if not same_user or rss:
|
||||
qpc.filter(models.Torrent.flags.op('&')(
|
||||
int(models.TorrentFlags.HIDDEN | models.TorrentFlags.ANONYMOUS)).is_(False))
|
||||
qpc.filter((models.Torrent.hidden == false()) &
|
||||
(models.Torrent.anonymous == false()))
|
||||
# General view (homepage, general search view)
|
||||
else:
|
||||
if not admin:
|
||||
# Hide all DELETED torrents if regular user
|
||||
qpc.filter(models.Torrent.flags.op('&')(
|
||||
int(models.TorrentFlags.DELETED)).is_(False))
|
||||
qpc.filter(models.Torrent.deleted == false())
|
||||
# If logged in, show all torrents that aren't hidden unless they belong to you
|
||||
# On RSS pages, show all public torrents and nothing more.
|
||||
if logged_in_user and not rss:
|
||||
qpc.filter(
|
||||
(models.Torrent.flags.op('&')(int(models.TorrentFlags.HIDDEN)).is_(False)) |
|
||||
(models.Torrent.uploader_id == logged_in_user.id))
|
||||
qpc.filter((models.Torrent.hidden == false()) |
|
||||
(models.Torrent.uploader_id == logged_in_user.id))
|
||||
# Otherwise, show all torrents that aren't hidden
|
||||
else:
|
||||
qpc.filter(models.Torrent.flags.op('&')(
|
||||
int(models.TorrentFlags.HIDDEN)).is_(False))
|
||||
qpc.filter(models.Torrent.hidden == false())
|
||||
|
||||
if main_category:
|
||||
qpc.filter(models.Torrent.main_category_id == main_cat_id)
|
||||
@@ -393,8 +390,7 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
|
||||
(models.Torrent.sub_category_id == sub_cat_id))
|
||||
|
||||
if filter_tuple:
|
||||
qpc.filter(models.Torrent.flags.op('&')(
|
||||
int(filter_tuple[0])).is_(filter_tuple[1]))
|
||||
qpc.filter(filter_tuple[0] == filter_tuple[1])
|
||||
|
||||
if term:
|
||||
for item in shlex.split(term, posix=False):
|
||||
@@ -403,11 +399,16 @@ def search_db(term='', user=None, sort='id', order='desc', category='0_0',
|
||||
item, models.TorrentNameSearch, FullTextMode.NATURAL))
|
||||
|
||||
query, count_query = qpc.items
|
||||
super_index = 'ix_' + models.Torrent._table_prefix('super')
|
||||
# Sort and order
|
||||
if sort_column.class_ != models.Torrent:
|
||||
index_name = _get_index_name(sort_column)
|
||||
query = query.join(sort_column.class_)
|
||||
query = query.with_hint(sort_column.class_, 'USE INDEX ({0})'.format(index_name))
|
||||
else:
|
||||
query = query.with_hint(models.Torrent, 'USE INDEX ({0})'.format(super_index))
|
||||
|
||||
count_query = count_query.with_hint(models.Torrent, 'USE INDEX ({0})'.format(super_index))
|
||||
|
||||
query = query.order_by(getattr(sort_column, order)())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user