mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 23:46:18 -05:00
Files are indexed by the words in their stem. Files can be searched via API.
This commit is contained in:
parent
30ca95d627
commit
a5e8ef16b6
10 changed files with 279 additions and 32 deletions
52
alembic/versions/36cae041ad31_files.py
Normal file
52
alembic/versions/36cae041ad31_files.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
"""Files
|
||||
|
||||
Revision ID: 36cae041ad31
|
||||
Revises: b97c1313f986
|
||||
Create Date: 2022-02-01 11:05:57.605541
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '36cae041ad31'
|
||||
down_revision = 'b97c1313f986'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('''CREATE TABLE file (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
title TEXT,
|
||||
mimetype TEXT,
|
||||
size INTEGER,
|
||||
modification_datetime TIMESTAMPTZ,
|
||||
textsearch tsvector GENERATED ALWAYS AS (
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(title, ''::text)), 'A'::"char")
|
||||
) stored
|
||||
)''')
|
||||
session.execute('CREATE INDEX file_path_idx ON file(path)')
|
||||
session.execute('CREATE INDEX file_mimetype_idx ON file(textsearch)')
|
||||
session.execute('CREATE INDEX file_textsearch_idx ON file(textsearch)')
|
||||
if not DOCKERIZED:
|
||||
session.execute('ALTER TABLE public.file OWNER TO wrolpi')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('DROP INDEX IF EXISTS file_textsearch_idx')
|
||||
session.execute('DROP INDEX IF EXISTS file_mimetype_idx')
|
||||
session.execute('DROP INDEX IF EXISTS file_path_idx')
|
||||
|
||||
session.execute('DROP TABLE IF EXISTS file')
|
||||
|
|
@ -7,7 +7,7 @@ from sanic_ext import validate
|
|||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi.common import get_media_directory, api_param_limiter
|
||||
from wrolpi.errors import InvalidFile, ValidationError
|
||||
from wrolpi.errors import InvalidFile
|
||||
from wrolpi.root_api import get_blueprint, json_response
|
||||
from . import lib, schema
|
||||
|
||||
|
|
@ -67,17 +67,15 @@ files_limit_limiter = api_param_limiter(100)
|
|||
files_offset_limiter = api_param_limiter(100, 0)
|
||||
|
||||
|
||||
@bp.post('/refresh')
|
||||
@openapi.description('Find and index all files')
|
||||
async def refresh(_: Request):
|
||||
lib.refresh_files()
|
||||
return response.empty()
|
||||
|
||||
|
||||
@bp.post('/search')
|
||||
@validate_doc(
|
||||
'Search files',
|
||||
consumes=schema.FilesSearchRequest,
|
||||
optional_body=True,
|
||||
)
|
||||
async def search(_: Request, data: dict):
|
||||
try:
|
||||
search_str = data.get('search_str')
|
||||
domain = data.get('domain')
|
||||
limit = files_limit_limiter(data.get('limit'))
|
||||
offset = files_offset_limiter(data.get('offset'))
|
||||
except Exception as e:
|
||||
raise ValidationError('Unable to validate search queries') from e
|
||||
@validate(schema.FilesSearchRequest)
|
||||
async def search_files(_: Request, body: schema.FilesSearchRequest):
|
||||
files, total = lib.search(body.search_str, body.limit, body.offset)
|
||||
return json_response(dict(files=files, totals=dict(files=total)))
|
||||
|
|
|
|||
|
|
@ -1,8 +1,19 @@
|
|||
import asyncio
|
||||
import re
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check
|
||||
import magic
|
||||
import psycopg2
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.files.models import File
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks
|
||||
from wrolpi.dates import from_timestamp
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
|
||||
from wrolpi.errors import InvalidFile
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
|
||||
def filter_parent_directories(directories: List[Path]) -> List[Path]:
|
||||
|
|
@ -56,3 +67,86 @@ def delete_file(file: str):
|
|||
if file.is_dir() or not file.is_file():
|
||||
raise InvalidFile(f'Invalid file {file}')
|
||||
file.unlink()
|
||||
|
||||
|
||||
FILE_NAME_REGEX = re.compile(r'[_ ]')
|
||||
|
||||
|
||||
def split_file_name(path: Path) -> List[str]:
|
||||
"""
|
||||
Split a file name into words.
|
||||
|
||||
>>> split_file_name(Path('foo.mp4'))
|
||||
['foo']
|
||||
>>> split_file_name(Path('foo bar.mp4'))
|
||||
['foo', 'bar']
|
||||
"""
|
||||
words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)]
|
||||
return words
|
||||
|
||||
|
||||
def upsert_file(path: Path, session: Session) -> File:
|
||||
file = session.query(File).filter_by(path=path).one_or_none()
|
||||
if not file:
|
||||
file = File(path=path)
|
||||
session.add(file)
|
||||
if not file.mimetype:
|
||||
file.mimetype = magic.from_file(str(path), mime=True)
|
||||
if not file.title:
|
||||
file.title = split_file_name(path)
|
||||
if not file.size or not file.modification_datetime:
|
||||
stat = path.stat()
|
||||
file.size = stat.st_size
|
||||
file.modification_datetime = file.modification_datetime or from_timestamp(stat.st_mtime)
|
||||
|
||||
return file
|
||||
|
||||
|
||||
def _refresh_files():
|
||||
"""Find and index all files"""
|
||||
paths = filter(lambda i: i.is_file(), walk(get_media_directory()))
|
||||
for chunk in chunks(paths, 20):
|
||||
with get_db_session(commit=True) as session:
|
||||
for path in chunk:
|
||||
upsert_file(path, session)
|
||||
|
||||
|
||||
@wraps(_refresh_files)
|
||||
def refresh_files():
|
||||
"""
|
||||
Schedule a refresh task if not testing. If testing, do a synchronous refresh.
|
||||
"""
|
||||
if PYTEST:
|
||||
return _refresh_files()
|
||||
|
||||
async def _():
|
||||
return _refresh_files()
|
||||
|
||||
asyncio.create_task(_())
|
||||
|
||||
|
||||
def search(search_str: str, limit: int, offset: int):
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total
|
||||
FROM file
|
||||
WHERE textsearch @@ websearch_to_tsquery(%(search_str)s)
|
||||
OFFSET %(offset)s LIMIT %(limit)s
|
||||
'''
|
||||
params = dict(search_str=search_str, offset=offset, limit=limit)
|
||||
curs.execute(stmt, params)
|
||||
|
||||
try:
|
||||
results = list(map(dict, curs.fetchall()))
|
||||
except psycopg2.ProgrammingError:
|
||||
# No files.
|
||||
return [], 0
|
||||
|
||||
total = results[0]['total'] if results else 0
|
||||
ranked_ids = [i['id'] for i in results]
|
||||
|
||||
with get_db_session() as session:
|
||||
results = get_ranked_models(ranked_ids, File, session)
|
||||
results = [i.__json__() for i in results]
|
||||
|
||||
return results, total
|
||||
|
|
|
|||
34
modules/files/models.py
Normal file
34
modules/files/models.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from sqlalchemy import Integer, Column, String, Computed
|
||||
from sqlalchemy.orm import deferred
|
||||
|
||||
from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector
|
||||
from wrolpi.dates import TZDateTime
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
||||
|
||||
class File(ModelHelper, Base):
|
||||
__tablename__ = 'file'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
mimetype = Column(String)
|
||||
modification_datetime = Column(TZDateTime)
|
||||
path = Column(MediaPathType)
|
||||
size = Column(Integer)
|
||||
title = Column(String)
|
||||
|
||||
textsearch = deferred(
|
||||
Column(tsvector, Computed('''to_tsvector('english'::regconfig, (COALESCE(title, ''::text) || ' '::text))''')))
|
||||
|
||||
def __repr__(self):
|
||||
return f'<File id={self.id} path={self.path.relative_to(get_media_directory())} mime={self.mimetype}>'
|
||||
|
||||
def __json__(self):
|
||||
d = dict(
|
||||
id=self.id,
|
||||
mimetype=self.mimetype,
|
||||
modification_datetime=self.modification_datetime,
|
||||
path=self.path.path.relative_to(get_media_directory()),
|
||||
size=self.size,
|
||||
title=self.title,
|
||||
)
|
||||
return d
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -12,7 +12,8 @@ class DeleteRequest:
|
|||
file: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class FilesSearchRequest:
|
||||
search_str = doc.String()
|
||||
limit = doc.Integer()
|
||||
offset = doc.Integer()
|
||||
search_str: str
|
||||
limit: Optional[int] = 20
|
||||
offset: Optional[int] = 0
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ from unittest import mock
|
|||
import pytest
|
||||
|
||||
from wrolpi.errors import API_ERRORS, WROLModeEnabled
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
def test_list_files_api(test_client, make_files_structure, test_directory):
|
||||
|
|
@ -114,3 +116,41 @@ def test_delete_wrol_mode(test_client):
|
|||
request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'foo'}))
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN
|
||||
assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
|
||||
|
||||
def do_search(test_client, search_str, total, expected):
|
||||
content = json.dumps({'search_str': search_str})
|
||||
request, response = test_client.post('/api/files/search', content=content)
|
||||
assert response.json['totals']['files'] == total
|
||||
for file, expected in zip_longest(response.json['files'], expected):
|
||||
assert_dict_contains(file, expected)
|
||||
|
||||
|
||||
def test_files_search(test_session, test_client, make_files_structure):
|
||||
# You can search an empty directory.
|
||||
do_search(test_client, 'nothing', 0, [])
|
||||
|
||||
files = [
|
||||
'foo_is_the_name.txt',
|
||||
'archives/bar.txt',
|
||||
'baz.mp4',
|
||||
'baz two.mp4'
|
||||
]
|
||||
foo, bar, baz, baz2 = make_files_structure(files)
|
||||
foo.write_text('foo contents')
|
||||
bar.write_text('the bar contents')
|
||||
baz.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
|
||||
baz2.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
|
||||
|
||||
# Refresh so files can be searched.
|
||||
request, response = test_client.post('/api/files/refresh')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)])
|
||||
do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)])
|
||||
do_search(test_client, 'baz', 2, [
|
||||
dict(path='baz two.mp4', mimetype='video/mp4', size=1055736),
|
||||
dict(path='baz.mp4', mimetype='video/mp4', size=1055736),
|
||||
])
|
||||
do_search(test_client, 'two', 1, [dict(path='baz two.mp4', mimetype='video/mp4', size=1055736)])
|
||||
do_search(test_client, 'nothing', 0, [])
|
||||
|
|
|
|||
|
|
@ -122,3 +122,18 @@ def test_delete_file(make_files_structure, test_directory):
|
|||
lib.delete_file('does not exist')
|
||||
|
||||
assert (test_directory / 'baz').is_dir()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'path,expected',
|
||||
[
|
||||
('foo.mp4', ['foo']),
|
||||
('foo bar.txt', ['foo', 'bar']),
|
||||
('foo_bar.mp4', ['foo', 'bar']),
|
||||
('foo-bar', ['foo-bar']),
|
||||
('123 foo bar', ['123', 'foo', 'bar']),
|
||||
('123foo bar', ['123foo', 'bar']),
|
||||
]
|
||||
)
|
||||
def test_split_file_name(path, expected):
|
||||
assert lib.split_file_name(Path(path)) == expected
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ Pint==0.18
|
|||
psycopg2-binary==2.9.3
|
||||
pytest-asyncio==0.17.2
|
||||
pytest==6.2.5
|
||||
python-magic==0.4.24
|
||||
pytz==2021.3
|
||||
PyYAML==6.0
|
||||
requests==2.27.1
|
||||
|
|
|
|||
|
|
@ -638,3 +638,11 @@ def zig_zag(low: ZIG_TYPE, high: ZIG_TYPE) -> Generator[ZIG_TYPE, None, None]:
|
|||
if num >= high:
|
||||
divisor *= 2
|
||||
num = low + (diff / divisor)
|
||||
|
||||
|
||||
def walk(path: Path) -> Generator[Path, None, None]:
|
||||
"""Walk a directory structure yielding all files and directories."""
|
||||
for path in path.iterdir():
|
||||
yield path
|
||||
if path.is_dir():
|
||||
yield from walk(path)
|
||||
|
|
|
|||
|
|
@ -76,19 +76,7 @@ class PytestCase:
|
|||
|
||||
@classmethod
|
||||
def assertDictContains(cls, d1: dict, d2: dict): # noqa
|
||||
if hasattr(d1, '__dict__'):
|
||||
d1 = d1.__dict__
|
||||
if hasattr(d2, '__dict__'):
|
||||
d2 = d2.__dict__
|
||||
|
||||
for k2 in d2.keys():
|
||||
assert d1, f'dict 1 is empty: {d1}'
|
||||
assert d2, f'dict 1 is empty: {d2}'
|
||||
assert k2 in d1, f'dict 1 does not contain {k2}'
|
||||
if isinstance(d1[k2], dict):
|
||||
cls.assertDictContains(d1[k2], d2[k2])
|
||||
else:
|
||||
assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'
|
||||
assert_dict_contains(d1, d2)
|
||||
|
||||
def assertError(self, response, http_status: int, code=None): # noqa
|
||||
self.assertEqual(response.status_code, http_status)
|
||||
|
|
@ -217,3 +205,19 @@ async def get_all_ws_messages(ws) -> List[dict]:
|
|||
break
|
||||
messages.append(json.loads(message))
|
||||
return messages
|
||||
|
||||
|
||||
def assert_dict_contains(d1: dict, d2: dict):
|
||||
if hasattr(d1, '__dict__'):
|
||||
d1 = d1.__dict__
|
||||
if hasattr(d2, '__dict__'):
|
||||
d2 = d2.__dict__
|
||||
|
||||
for k2 in d2.keys():
|
||||
assert d1, f'dict 1 is empty: {d1}'
|
||||
assert d2, f'dict 1 is empty: {d2}'
|
||||
assert k2 in d1, f'dict 1 does not contain {k2}'
|
||||
if isinstance(d1[k2], dict):
|
||||
assert_dict_contains(d1[k2], d2[k2])
|
||||
else:
|
||||
assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue