From a5e8ef16b6a53b42731175b2899c06aeff0f9e1b Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 1 Feb 2022 12:22:29 -0700 Subject: [PATCH] Files are indexed by the words in their stem. Files can be searched via API. --- alembic/versions/36cae041ad31_files.py | 52 ++++++++++++++ modules/files/api.py | 26 ++++--- modules/files/lib.py | 96 +++++++++++++++++++++++++- modules/files/models.py | 34 +++++++++ modules/files/schema.py | 9 +-- modules/files/test/test_api.py | 40 +++++++++++ modules/files/test/test_lib.py | 15 ++++ requirements.txt | 1 + wrolpi/common.py | 8 +++ wrolpi/test/common.py | 30 ++++---- 10 files changed, 279 insertions(+), 32 deletions(-) create mode 100644 alembic/versions/36cae041ad31_files.py create mode 100644 modules/files/models.py diff --git a/alembic/versions/36cae041ad31_files.py b/alembic/versions/36cae041ad31_files.py new file mode 100644 index 00000000..dc8f8a2e --- /dev/null +++ b/alembic/versions/36cae041ad31_files.py @@ -0,0 +1,52 @@ +"""Files + +Revision ID: 36cae041ad31 +Revises: b97c1313f986 +Create Date: 2022-02-01 11:05:57.605541 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '36cae041ad31' +down_revision = 'b97c1313f986' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('''CREATE TABLE file ( + id BIGSERIAL PRIMARY KEY, + path TEXT NOT NULL, + title TEXT, + mimetype TEXT, + size INTEGER, + modification_datetime TIMESTAMPTZ, + textsearch tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english'::regconfig, COALESCE(title, ''::text)), 'A'::"char") + ) stored + )''') + session.execute('CREATE INDEX file_path_idx ON file(path)') + session.execute('CREATE INDEX file_mimetype_idx ON file(textsearch)') + session.execute('CREATE INDEX file_textsearch_idx ON file(textsearch)') + if not DOCKERIZED: + session.execute('ALTER TABLE public.file OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('DROP INDEX IF EXISTS file_textsearch_idx') + session.execute('DROP INDEX IF EXISTS file_mimetype_idx') + session.execute('DROP INDEX IF EXISTS file_path_idx') + + session.execute('DROP TABLE IF EXISTS file') diff --git a/modules/files/api.py b/modules/files/api.py index 49905bd3..45c78f92 100644 --- a/modules/files/api.py +++ b/modules/files/api.py @@ -7,7 +7,7 @@ from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi.common import get_media_directory, api_param_limiter -from wrolpi.errors import InvalidFile, ValidationError +from wrolpi.errors import InvalidFile from wrolpi.root_api import get_blueprint, json_response from . import lib, schema @@ -67,17 +67,15 @@ files_limit_limiter = api_param_limiter(100) files_offset_limiter = api_param_limiter(100, 0) +@bp.post('/refresh') +@openapi.description('Find and index all files') +async def refresh(_: Request): + lib.refresh_files() + return response.empty() + + @bp.post('/search') -@validate_doc( - 'Search files', - consumes=schema.FilesSearchRequest, - optional_body=True, -) -async def search(_: Request, data: dict): - try: - search_str = data.get('search_str') - domain = data.get('domain') - limit = files_limit_limiter(data.get('limit')) - offset = files_offset_limiter(data.get('offset')) - except Exception as e: - raise ValidationError('Unable to validate search queries') from e +@validate(schema.FilesSearchRequest) +async def search_files(_: Request, body: schema.FilesSearchRequest): + files, total = lib.search(body.search_str, body.limit, body.offset) + return json_response(dict(files=files, totals=dict(files=total))) diff --git a/modules/files/lib.py b/modules/files/lib.py index 6224be4e..b93f5fcf 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -1,8 +1,19 @@ +import asyncio +import re +from functools import wraps from pathlib import Path from typing import List -from wrolpi.common import get_media_directory, wrol_mode_check +import magic +import psycopg2 +from sqlalchemy.orm import Session + +from modules.files.models import File +from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks +from wrolpi.dates import from_timestamp +from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidFile +from wrolpi.vars import PYTEST def filter_parent_directories(directories: List[Path]) -> List[Path]: @@ -56,3 +67,86 @@ def delete_file(file: str): if file.is_dir() or not file.is_file(): raise InvalidFile(f'Invalid file {file}') file.unlink() + + +FILE_NAME_REGEX = re.compile(r'[_ ]') + + +def split_file_name(path: Path) -> List[str]: + """ + Split a file name into words. + + >>> split_file_name(Path('foo.mp4')) + ['foo'] + >>> split_file_name(Path('foo bar.mp4')) + ['foo', 'bar'] + """ + words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)] + return words + + +def upsert_file(path: Path, session: Session) -> File: + file = session.query(File).filter_by(path=path).one_or_none() + if not file: + file = File(path=path) + session.add(file) + if not file.mimetype: + file.mimetype = magic.from_file(str(path), mime=True) + if not file.title: + file.title = split_file_name(path) + if not file.size or not file.modification_datetime: + stat = path.stat() + file.size = stat.st_size + file.modification_datetime = file.modification_datetime or from_timestamp(stat.st_mtime) + + return file + + +def _refresh_files(): + """Find and index all files""" + paths = filter(lambda i: i.is_file(), walk(get_media_directory())) + for chunk in chunks(paths, 20): + with get_db_session(commit=True) as session: + for path in chunk: + upsert_file(path, session) + + +@wraps(_refresh_files) +def refresh_files(): + """ + Schedule a refresh task if not testing. If testing, do a synchronous refresh. + """ + if PYTEST: + return _refresh_files() + + async def _(): + return _refresh_files() + + asyncio.create_task(_()) + + +def search(search_str: str, limit: int, offset: int): + with get_db_curs() as curs: + stmt = ''' + SELECT id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total + FROM file + WHERE textsearch @@ websearch_to_tsquery(%(search_str)s) + OFFSET %(offset)s LIMIT %(limit)s + ''' + params = dict(search_str=search_str, offset=offset, limit=limit) + curs.execute(stmt, params) + + try: + results = list(map(dict, curs.fetchall())) + except psycopg2.ProgrammingError: + # No files. + return [], 0 + + total = results[0]['total'] if results else 0 + ranked_ids = [i['id'] for i in results] + + with get_db_session() as session: + results = get_ranked_models(ranked_ids, File, session) + results = [i.__json__() for i in results] + + return results, total diff --git a/modules/files/models.py b/modules/files/models.py new file mode 100644 index 00000000..979a5391 --- /dev/null +++ b/modules/files/models.py @@ -0,0 +1,34 @@ +from sqlalchemy import Integer, Column, String, Computed +from sqlalchemy.orm import deferred + +from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector +from wrolpi.dates import TZDateTime +from wrolpi.media_path import MediaPathType + + +class File(ModelHelper, Base): + __tablename__ = 'file' + id = Column(Integer, primary_key=True) + + mimetype = Column(String) + modification_datetime = Column(TZDateTime) + path = Column(MediaPathType) + size = Column(Integer) + title = Column(String) + + textsearch = deferred( + Column(tsvector, Computed('''to_tsvector('english'::regconfig, (COALESCE(title, ''::text) || ' '::text))'''))) + + def __repr__(self): + return f'' + + def __json__(self): + d = dict( + id=self.id, + mimetype=self.mimetype, + modification_datetime=self.modification_datetime, + path=self.path.path.relative_to(get_media_directory()), + size=self.size, + title=self.title, + ) + return d diff --git a/modules/files/schema.py b/modules/files/schema.py index 76800b21..af2bf73b 100644 --- a/modules/files/schema.py +++ b/modules/files/schema.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import List +from typing import List, Optional @dataclass @@ -12,7 +12,8 @@ class DeleteRequest: file: str +@dataclass class FilesSearchRequest: - search_str = doc.String() - limit = doc.Integer() - offset = doc.Integer() + search_str: str + limit: Optional[int] = 20 + offset: Optional[int] = 0 diff --git a/modules/files/test/test_api.py b/modules/files/test/test_api.py index 0fb1ea41..1b09e281 100644 --- a/modules/files/test/test_api.py +++ b/modules/files/test/test_api.py @@ -6,6 +6,8 @@ from unittest import mock import pytest from wrolpi.errors import API_ERRORS, WROLModeEnabled +from wrolpi.test.common import assert_dict_contains +from wrolpi.vars import PROJECT_DIR def test_list_files_api(test_client, make_files_structure, test_directory): @@ -114,3 +116,41 @@ def test_delete_wrol_mode(test_client): request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'foo'})) assert response.status_code == HTTPStatus.FORBIDDEN assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + +def do_search(test_client, search_str, total, expected): + content = json.dumps({'search_str': search_str}) + request, response = test_client.post('/api/files/search', content=content) + assert response.json['totals']['files'] == total + for file, expected in zip_longest(response.json['files'], expected): + assert_dict_contains(file, expected) + + +def test_files_search(test_session, test_client, make_files_structure): + # You can search an empty directory. + do_search(test_client, 'nothing', 0, []) + + files = [ + 'foo_is_the_name.txt', + 'archives/bar.txt', + 'baz.mp4', + 'baz two.mp4' + ] + foo, bar, baz, baz2 = make_files_structure(files) + foo.write_text('foo contents') + bar.write_text('the bar contents') + baz.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()) + baz2.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()) + + # Refresh so files can be searched. + request, response = test_client.post('/api/files/refresh') + assert response.status_code == HTTPStatus.NO_CONTENT + + do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)]) + do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)]) + do_search(test_client, 'baz', 2, [ + dict(path='baz two.mp4', mimetype='video/mp4', size=1055736), + dict(path='baz.mp4', mimetype='video/mp4', size=1055736), + ]) + do_search(test_client, 'two', 1, [dict(path='baz two.mp4', mimetype='video/mp4', size=1055736)]) + do_search(test_client, 'nothing', 0, []) diff --git a/modules/files/test/test_lib.py b/modules/files/test/test_lib.py index 1a53bb29..57088aed 100644 --- a/modules/files/test/test_lib.py +++ b/modules/files/test/test_lib.py @@ -122,3 +122,18 @@ def test_delete_file(make_files_structure, test_directory): lib.delete_file('does not exist') assert (test_directory / 'baz').is_dir() + + +@pytest.mark.parametrize( + 'path,expected', + [ + ('foo.mp4', ['foo']), + ('foo bar.txt', ['foo', 'bar']), + ('foo_bar.mp4', ['foo', 'bar']), + ('foo-bar', ['foo-bar']), + ('123 foo bar', ['123', 'foo', 'bar']), + ('123foo bar', ['123foo', 'bar']), + ] +) +def test_split_file_name(path, expected): + assert lib.split_file_name(Path(path)) == expected diff --git a/requirements.txt b/requirements.txt index 3127e2c4..e5954ff3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ Pint==0.18 psycopg2-binary==2.9.3 pytest-asyncio==0.17.2 pytest==6.2.5 +python-magic==0.4.24 pytz==2021.3 PyYAML==6.0 requests==2.27.1 diff --git a/wrolpi/common.py b/wrolpi/common.py index 19134a0a..9056e473 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -638,3 +638,11 @@ def zig_zag(low: ZIG_TYPE, high: ZIG_TYPE) -> Generator[ZIG_TYPE, None, None]: if num >= high: divisor *= 2 num = low + (diff / divisor) + + +def walk(path: Path) -> Generator[Path, None, None]: + """Walk a directory structure yielding all files and directories.""" + for path in path.iterdir(): + yield path + if path.is_dir(): + yield from walk(path) diff --git a/wrolpi/test/common.py b/wrolpi/test/common.py index a93bc3dd..efbdd4a5 100644 --- a/wrolpi/test/common.py +++ b/wrolpi/test/common.py @@ -76,19 +76,7 @@ class PytestCase: @classmethod def assertDictContains(cls, d1: dict, d2: dict): # noqa - if hasattr(d1, '__dict__'): - d1 = d1.__dict__ - if hasattr(d2, '__dict__'): - d2 = d2.__dict__ - - for k2 in d2.keys(): - assert d1, f'dict 1 is empty: {d1}' - assert d2, f'dict 1 is empty: {d2}' - assert k2 in d1, f'dict 1 does not contain {k2}' - if isinstance(d1[k2], dict): - cls.assertDictContains(d1[k2], d2[k2]) - else: - assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1' + assert_dict_contains(d1, d2) def assertError(self, response, http_status: int, code=None): # noqa self.assertEqual(response.status_code, http_status) @@ -217,3 +205,19 @@ async def get_all_ws_messages(ws) -> List[dict]: break messages.append(json.loads(message)) return messages + + +def assert_dict_contains(d1: dict, d2: dict): + if hasattr(d1, '__dict__'): + d1 = d1.__dict__ + if hasattr(d2, '__dict__'): + d2 = d2.__dict__ + + for k2 in d2.keys(): + assert d1, f'dict 1 is empty: {d1}' + assert d2, f'dict 1 is empty: {d2}' + assert k2 in d1, f'dict 1 does not contain {k2}' + if isinstance(d1[k2], dict): + assert_dict_contains(d1[k2], d2[k2]) + else: + assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'