mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
Search suggestions handles any combination of filtering.
This commit is contained in:
parent
23d84ed107
commit
8916600a14
4 changed files with 89 additions and 49 deletions
|
|
@ -91,8 +91,9 @@ def video_factory(test_session, test_directory):
|
|||
|
||||
def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file=None,
|
||||
with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False,
|
||||
source_id: str = None) -> Video:
|
||||
source_id: str = None, tag_names: List[str] = None) -> Video:
|
||||
title = title or str(uuid4())
|
||||
tag_names = tag_names or list()
|
||||
|
||||
if with_video_file and isinstance(with_video_file, (pathlib.Path, str)):
|
||||
# Put the video exactly where specified for the test.
|
||||
|
|
@ -134,6 +135,10 @@ def video_factory(test_session, test_directory):
|
|||
video.source_id = source_id or title
|
||||
video.file_group.published_datetime = upload_date
|
||||
video.validate()
|
||||
|
||||
for tag_name in tag_names:
|
||||
video.add_tag(tag_name)
|
||||
|
||||
return video
|
||||
|
||||
return factory
|
||||
|
|
|
|||
|
|
@ -46,7 +46,8 @@ except ImportError:
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
|
||||
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', 'estimate_search',
|
||||
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
|
||||
'search_file_suggestion_count',
|
||||
'move', 'rename', 'delete_directory', 'handle_file_group_search_results']
|
||||
|
||||
|
||||
|
|
@ -709,7 +710,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
wheres.append('model = %(model)s')
|
||||
|
||||
if tags:
|
||||
# Filter all FileGroups by those that have been tagged with the provided tag names.
|
||||
# Filter all FileGroups by those that have been tagged with all the provided tag names.
|
||||
tags_stmt, params_ = tag_names_to_file_group_sub_select(tags)
|
||||
params.update(params_)
|
||||
wheres.append(f'fg.id = ANY({tags_stmt})')
|
||||
|
|
@ -1054,42 +1055,49 @@ def get_refresh_progress() -> RefreshProgress:
|
|||
|
||||
def mimetypes_to_sql_wheres(mimetypes: List[str], params: dict):
|
||||
if mimetypes:
|
||||
mimetype_wheres = []
|
||||
wheres = list()
|
||||
for idx, mimetype in enumerate(mimetypes):
|
||||
key = f'mimetype{idx}'
|
||||
if len(mimetype.split('/')) == 1:
|
||||
params[key] = f'{mimetype}/%'
|
||||
else:
|
||||
params[key] = f'{mimetype}%'
|
||||
mimetype_wheres.append(f'mimetype LIKE %({key})s')
|
||||
mimetype_wheres = ' OR '.join(mimetype_wheres)
|
||||
return f'({mimetype_wheres})', params
|
||||
wheres.append(f'mimetype LIKE %({key})s')
|
||||
return f'({" OR ".join(wheres)})', params
|
||||
return '', params
|
||||
|
||||
|
||||
async def estimate_search(search_str: str, tag_names: List[str], mimetypes: List[str]):
|
||||
async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str]):
|
||||
"""
|
||||
Return FileGroup count of what will be returned if the search is actually performed.
|
||||
"""
|
||||
wheres = ['textsearch @@ websearch_to_tsquery(%(search_str)s)']
|
||||
wheres = []
|
||||
joins = list()
|
||||
params = dict(search_str=search_str, tag_names=tag_names, mimetypes=mimetypes)
|
||||
group_by = ''
|
||||
having = ''
|
||||
|
||||
if search_str:
|
||||
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
if tag_names:
|
||||
joins.append('LEFT JOIN tag_file tf on file_group.id = tf.file_group_id')
|
||||
wheres.append('tf.tag_id IN (select id from tag where name = ANY(%(tag_names)s))')
|
||||
# Only estimate files that have been tagged with all tag names.
|
||||
tags_stmt, params_ = tag_names_to_file_group_sub_select(tag_names)
|
||||
params.update(params_)
|
||||
wheres.append(f'fg.id = ANY({tags_stmt})')
|
||||
|
||||
mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params)
|
||||
if mimetype_wheres:
|
||||
wheres.append(mimetype_wheres)
|
||||
|
||||
joins = "\n".join(joins)
|
||||
wheres = "\nAND ".join(wheres)
|
||||
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
|
||||
stmt = f'''
|
||||
SELECT COUNT(*) OVER() AS estimate
|
||||
FROM file_group
|
||||
FROM file_group fg
|
||||
{joins}
|
||||
WHERE {wheres}
|
||||
{wheres}
|
||||
{group_by}
|
||||
{having}
|
||||
'''
|
||||
logger.debug(stmt, params)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from wrolpi.downloader import download_manager
|
|||
from wrolpi.errors import WROLModeEnabled, APIError, HotspotError, InvalidDownload, \
|
||||
HotspotPasswordTooShort, NativeOnly, InvalidConfig
|
||||
from wrolpi.events import get_events, Events
|
||||
from wrolpi.files.lib import get_file_statistics, estimate_search
|
||||
from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
|
||||
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
|
||||
truthy_arg
|
||||
from wrolpi.version import __version__
|
||||
|
|
@ -514,7 +514,7 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ
|
|||
from modules.archive.lib import search_domains_by_name
|
||||
|
||||
file_groups, channels, domains = await asyncio.gather(
|
||||
estimate_search(body.search_str, body.tag_names, body.mimetypes),
|
||||
search_file_suggestion_count(body.search_str, body.tag_names, body.mimetypes),
|
||||
search_channels_by_name(body.search_str),
|
||||
search_domains_by_name(body.search_str),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -474,47 +474,74 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
|
|||
channel_factory(name='Foo')
|
||||
channel_factory(name='Fool')
|
||||
channel_factory(name='Bar')
|
||||
archive_factory(domain='foo.com', contents='contents of foo with bunny')
|
||||
archive_factory(domain='bar.com', contents='contents of bar')
|
||||
video_factory(with_caption_file=True)
|
||||
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
|
||||
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
||||
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
|
||||
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
|
||||
test_session.commit()
|
||||
|
||||
body = dict(search_str='foo')
|
||||
request, response = await test_async_client.post('/api/search_suggestions', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['channels'] == [{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo'},
|
||||
{'directory': 'Fool',
|
||||
'id': 2,
|
||||
'name': 'Fool',
|
||||
'url': 'https://example.com/Fool'}]
|
||||
assert response.json['domains'] == [{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}]
|
||||
assert response.json['file_groups'] == 1
|
||||
async def assert_results(body: dict, expected_channels=None, expected_domains=None, expected_file_groups=None):
|
||||
expected_channels = expected_channels or []
|
||||
expected_domains = expected_domains or []
|
||||
expected_file_groups = expected_file_groups or 0
|
||||
|
||||
body = dict(search_str='bar')
|
||||
request, response = await test_async_client.post('/api/search_suggestions', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['channels'] == [{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}]
|
||||
assert response.json['domains'] == [{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}]
|
||||
assert response.json['file_groups'] == 1
|
||||
request, response = await test_async_client.post('/api/search_suggestions', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
if expected_channels:
|
||||
assert response.json['channels'] == expected_channels
|
||||
assert response.json['domains'] == expected_domains
|
||||
assert response.json['file_groups'] == expected_file_groups or 0
|
||||
|
||||
body = dict(search_str='one')
|
||||
request, response = await test_async_client.post('/api/search_suggestions', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['channels'] == []
|
||||
assert response.json['domains'] == []
|
||||
assert response.json['file_groups'] == 0
|
||||
await assert_results(
|
||||
dict(search_str='foo'),
|
||||
[{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo'},
|
||||
{'directory': 'Fool',
|
||||
'id': 2,
|
||||
'name': 'Fool',
|
||||
'url': 'https://example.com/Fool'}],
|
||||
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}],
|
||||
1,
|
||||
)
|
||||
|
||||
await assert_results(
|
||||
dict(search_str='bar'),
|
||||
[{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}],
|
||||
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}],
|
||||
1,
|
||||
)
|
||||
|
||||
# "foo" archive and video both contain bunny.
|
||||
body = dict(search_str='bunny')
|
||||
request, response = await test_async_client.post('/api/search_suggestions', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file_groups'] == 2
|
||||
await assert_results(dict(search_str='bunny'), expected_file_groups=2)
|
||||
|
||||
# Filtering with mimetypes removes archive result.
|
||||
body = dict(search_str='bunny', mimetypes=['video'])
|
||||
request, response = await test_async_client.post('/api/search_suggestions', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file_groups'] == 1
|
||||
await assert_results(dict(search_str='bunny', mimetypes=['video']), expected_file_groups=1)
|
||||
|
||||
# tag1 has been used twice
|
||||
await assert_results(dict(tag_names=[tag1.name, ]), expected_file_groups=2)
|
||||
|
||||
# archive2 was tagged with tag2, but only video contains bunny.
|
||||
await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), expected_file_groups=1)
|
||||
|
||||
# tag2 has been used twice
|
||||
await assert_results(dict(tag_names=[tag2.name, ]), expected_file_groups=2)
|
||||
|
||||
# tag2 has been used twice, but filter by video mimetype.
|
||||
await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), expected_file_groups=1)
|
||||
|
||||
# Can use all filters simultaneously.
|
||||
await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']),
|
||||
expected_file_groups=1)
|
||||
|
||||
# Only the video has been tagged with both tag1 and tag2.
|
||||
await assert_results(dict(tag_names=[tag1.name, tag2.name]), expected_file_groups=1)
|
||||
|
||||
# tag3 was never used
|
||||
await assert_results(dict(tag_names=[tag3.name]))
|
||||
|
||||
await assert_results(dict(search_str='Does not exist.'))
|
||||
|
||||
# No PDFs.
|
||||
await assert_results(dict(mimetypes=['application/pdf']))
|
||||
|
||||
|
||||
def test_recursive_errors():
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue