Files are indexed by the words in their stem. Files can be searched via API.

This commit is contained in:
Roland 2022-02-01 12:22:29 -07:00
parent 30ca95d627
commit a5e8ef16b6
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
10 changed files with 279 additions and 32 deletions

View file

@ -0,0 +1,52 @@
"""Files
Revision ID: 36cae041ad31
Revises: b97c1313f986
Create Date: 2022-02-01 11:05:57.605541
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '36cae041ad31'
down_revision = 'b97c1313f986'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('''CREATE TABLE file (
id BIGSERIAL PRIMARY KEY,
path TEXT NOT NULL,
title TEXT,
mimetype TEXT,
size INTEGER,
modification_datetime TIMESTAMPTZ,
textsearch tsvector GENERATED ALWAYS AS (
setweight(to_tsvector('english'::regconfig, COALESCE(title, ''::text)), 'A'::"char")
) stored
)''')
session.execute('CREATE INDEX file_path_idx ON file(path)')
session.execute('CREATE INDEX file_mimetype_idx ON file(textsearch)')
session.execute('CREATE INDEX file_textsearch_idx ON file(textsearch)')
if not DOCKERIZED:
session.execute('ALTER TABLE public.file OWNER TO wrolpi')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP INDEX IF EXISTS file_textsearch_idx')
session.execute('DROP INDEX IF EXISTS file_mimetype_idx')
session.execute('DROP INDEX IF EXISTS file_path_idx')
session.execute('DROP TABLE IF EXISTS file')

View file

@ -7,7 +7,7 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import get_media_directory, api_param_limiter
from wrolpi.errors import InvalidFile, ValidationError
from wrolpi.errors import InvalidFile
from wrolpi.root_api import get_blueprint, json_response
from . import lib, schema
@ -67,17 +67,15 @@ files_limit_limiter = api_param_limiter(100)
files_offset_limiter = api_param_limiter(100, 0)
@bp.post('/refresh')
@openapi.description('Find and index all files')
async def refresh(_: Request):
lib.refresh_files()
return response.empty()
@bp.post('/search')
@validate_doc(
'Search files',
consumes=schema.FilesSearchRequest,
optional_body=True,
)
async def search(_: Request, data: dict):
try:
search_str = data.get('search_str')
domain = data.get('domain')
limit = files_limit_limiter(data.get('limit'))
offset = files_offset_limiter(data.get('offset'))
except Exception as e:
raise ValidationError('Unable to validate search queries') from e
@validate(schema.FilesSearchRequest)
async def search_files(_: Request, body: schema.FilesSearchRequest):
files, total = lib.search(body.search_str, body.limit, body.offset)
return json_response(dict(files=files, totals=dict(files=total)))

View file

@ -1,8 +1,19 @@
import asyncio
import re
from functools import wraps
from pathlib import Path
from typing import List
from wrolpi.common import get_media_directory, wrol_mode_check
import magic
import psycopg2
from sqlalchemy.orm import Session
from modules.files.models import File
from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks
from wrolpi.dates import from_timestamp
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
from wrolpi.errors import InvalidFile
from wrolpi.vars import PYTEST
def filter_parent_directories(directories: List[Path]) -> List[Path]:
@ -56,3 +67,86 @@ def delete_file(file: str):
if file.is_dir() or not file.is_file():
raise InvalidFile(f'Invalid file {file}')
file.unlink()
FILE_NAME_REGEX = re.compile(r'[_ ]')
def split_file_name(path: Path) -> List[str]:
"""
Split a file name into words.
>>> split_file_name(Path('foo.mp4'))
['foo']
>>> split_file_name(Path('foo bar.mp4'))
['foo', 'bar']
"""
words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)]
return words
def upsert_file(path: Path, session: Session) -> File:
file = session.query(File).filter_by(path=path).one_or_none()
if not file:
file = File(path=path)
session.add(file)
if not file.mimetype:
file.mimetype = magic.from_file(str(path), mime=True)
if not file.title:
file.title = split_file_name(path)
if not file.size or not file.modification_datetime:
stat = path.stat()
file.size = stat.st_size
file.modification_datetime = file.modification_datetime or from_timestamp(stat.st_mtime)
return file
def _refresh_files():
"""Find and index all files"""
paths = filter(lambda i: i.is_file(), walk(get_media_directory()))
for chunk in chunks(paths, 20):
with get_db_session(commit=True) as session:
for path in chunk:
upsert_file(path, session)
@wraps(_refresh_files)
def refresh_files():
"""
Schedule a refresh task if not testing. If testing, do a synchronous refresh.
"""
if PYTEST:
return _refresh_files()
async def _():
return _refresh_files()
asyncio.create_task(_())
def search(search_str: str, limit: int, offset: int):
with get_db_curs() as curs:
stmt = '''
SELECT id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total
FROM file
WHERE textsearch @@ websearch_to_tsquery(%(search_str)s)
OFFSET %(offset)s LIMIT %(limit)s
'''
params = dict(search_str=search_str, offset=offset, limit=limit)
curs.execute(stmt, params)
try:
results = list(map(dict, curs.fetchall()))
except psycopg2.ProgrammingError:
# No files.
return [], 0
total = results[0]['total'] if results else 0
ranked_ids = [i['id'] for i in results]
with get_db_session() as session:
results = get_ranked_models(ranked_ids, File, session)
results = [i.__json__() for i in results]
return results, total

34
modules/files/models.py Normal file
View file

@ -0,0 +1,34 @@
from sqlalchemy import Integer, Column, String, Computed
from sqlalchemy.orm import deferred
from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector
from wrolpi.dates import TZDateTime
from wrolpi.media_path import MediaPathType
class File(ModelHelper, Base):
__tablename__ = 'file'
id = Column(Integer, primary_key=True)
mimetype = Column(String)
modification_datetime = Column(TZDateTime)
path = Column(MediaPathType)
size = Column(Integer)
title = Column(String)
textsearch = deferred(
Column(tsvector, Computed('''to_tsvector('english'::regconfig, (COALESCE(title, ''::text) || ' '::text))''')))
def __repr__(self):
return f'<File id={self.id} path={self.path.relative_to(get_media_directory())} mime={self.mimetype}>'
def __json__(self):
d = dict(
id=self.id,
mimetype=self.mimetype,
modification_datetime=self.modification_datetime,
path=self.path.path.relative_to(get_media_directory()),
size=self.size,
title=self.title,
)
return d

View file

@ -1,5 +1,5 @@
from dataclasses import dataclass, field
from typing import List
from typing import List, Optional
@dataclass
@ -12,7 +12,8 @@ class DeleteRequest:
file: str
@dataclass
class FilesSearchRequest:
search_str = doc.String()
limit = doc.Integer()
offset = doc.Integer()
search_str: str
limit: Optional[int] = 20
offset: Optional[int] = 0

View file

@ -6,6 +6,8 @@ from unittest import mock
import pytest
from wrolpi.errors import API_ERRORS, WROLModeEnabled
from wrolpi.test.common import assert_dict_contains
from wrolpi.vars import PROJECT_DIR
def test_list_files_api(test_client, make_files_structure, test_directory):
@ -114,3 +116,41 @@ def test_delete_wrol_mode(test_client):
request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'foo'}))
assert response.status_code == HTTPStatus.FORBIDDEN
assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code']
def do_search(test_client, search_str, total, expected):
content = json.dumps({'search_str': search_str})
request, response = test_client.post('/api/files/search', content=content)
assert response.json['totals']['files'] == total
for file, expected in zip_longest(response.json['files'], expected):
assert_dict_contains(file, expected)
def test_files_search(test_session, test_client, make_files_structure):
# You can search an empty directory.
do_search(test_client, 'nothing', 0, [])
files = [
'foo_is_the_name.txt',
'archives/bar.txt',
'baz.mp4',
'baz two.mp4'
]
foo, bar, baz, baz2 = make_files_structure(files)
foo.write_text('foo contents')
bar.write_text('the bar contents')
baz.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
baz2.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
# Refresh so files can be searched.
request, response = test_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)])
do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)])
do_search(test_client, 'baz', 2, [
dict(path='baz two.mp4', mimetype='video/mp4', size=1055736),
dict(path='baz.mp4', mimetype='video/mp4', size=1055736),
])
do_search(test_client, 'two', 1, [dict(path='baz two.mp4', mimetype='video/mp4', size=1055736)])
do_search(test_client, 'nothing', 0, [])

View file

@ -122,3 +122,18 @@ def test_delete_file(make_files_structure, test_directory):
lib.delete_file('does not exist')
assert (test_directory / 'baz').is_dir()
@pytest.mark.parametrize(
'path,expected',
[
('foo.mp4', ['foo']),
('foo bar.txt', ['foo', 'bar']),
('foo_bar.mp4', ['foo', 'bar']),
('foo-bar', ['foo-bar']),
('123 foo bar', ['123', 'foo', 'bar']),
('123foo bar', ['123foo', 'bar']),
]
)
def test_split_file_name(path, expected):
assert lib.split_file_name(Path(path)) == expected

View file

@ -7,6 +7,7 @@ Pint==0.18
psycopg2-binary==2.9.3
pytest-asyncio==0.17.2
pytest==6.2.5
python-magic==0.4.24
pytz==2021.3
PyYAML==6.0
requests==2.27.1

View file

@ -638,3 +638,11 @@ def zig_zag(low: ZIG_TYPE, high: ZIG_TYPE) -> Generator[ZIG_TYPE, None, None]:
if num >= high:
divisor *= 2
num = low + (diff / divisor)
def walk(path: Path) -> Generator[Path, None, None]:
"""Walk a directory structure yielding all files and directories."""
for path in path.iterdir():
yield path
if path.is_dir():
yield from walk(path)

View file

@ -76,19 +76,7 @@ class PytestCase:
@classmethod
def assertDictContains(cls, d1: dict, d2: dict): # noqa
if hasattr(d1, '__dict__'):
d1 = d1.__dict__
if hasattr(d2, '__dict__'):
d2 = d2.__dict__
for k2 in d2.keys():
assert d1, f'dict 1 is empty: {d1}'
assert d2, f'dict 1 is empty: {d2}'
assert k2 in d1, f'dict 1 does not contain {k2}'
if isinstance(d1[k2], dict):
cls.assertDictContains(d1[k2], d2[k2])
else:
assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'
assert_dict_contains(d1, d2)
def assertError(self, response, http_status: int, code=None): # noqa
self.assertEqual(response.status_code, http_status)
@ -217,3 +205,19 @@ async def get_all_ws_messages(ws) -> List[dict]:
break
messages.append(json.loads(message))
return messages
def assert_dict_contains(d1: dict, d2: dict):
if hasattr(d1, '__dict__'):
d1 = d1.__dict__
if hasattr(d2, '__dict__'):
d2 = d2.__dict__
for k2 in d2.keys():
assert d1, f'dict 1 is empty: {d1}'
assert d2, f'dict 1 is empty: {d2}'
assert k2 in d1, f'dict 1 does not contain {k2}'
if isinstance(d1[k2], dict):
assert_dict_contains(d1[k2], d2[k2])
else:
assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'