Search suggestions handles any combination of filtering.

This commit is contained in:
Roland 2024-02-10 19:42:29 -07:00
parent 23d84ed107
commit 8916600a14
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
4 changed files with 89 additions and 49 deletions

View file

@ -91,8 +91,9 @@ def video_factory(test_session, test_directory):
def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file=None,
with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False,
source_id: str = None) -> Video:
source_id: str = None, tag_names: List[str] = None) -> Video:
title = title or str(uuid4())
tag_names = tag_names or list()
if with_video_file and isinstance(with_video_file, (pathlib.Path, str)):
# Put the video exactly where specified for the test.
@ -134,6 +135,10 @@ def video_factory(test_session, test_directory):
video.source_id = source_id or title
video.file_group.published_datetime = upload_date
video.validate()
for tag_name in tag_names:
video.add_tag(tag_name)
return video
return factory

View file

@ -46,7 +46,8 @@ except ImportError:
logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', 'estimate_search',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
'search_file_suggestion_count',
'move', 'rename', 'delete_directory', 'handle_file_group_search_results']
@ -709,7 +710,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
wheres.append('model = %(model)s')
if tags:
# Filter all FileGroups by those that have been tagged with the provided tag names.
# Filter all FileGroups by those that have been tagged with all the provided tag names.
tags_stmt, params_ = tag_names_to_file_group_sub_select(tags)
params.update(params_)
wheres.append(f'fg.id = ANY({tags_stmt})')
@ -1054,42 +1055,49 @@ def get_refresh_progress() -> RefreshProgress:
def mimetypes_to_sql_wheres(mimetypes: List[str], params: dict):
if mimetypes:
mimetype_wheres = []
wheres = list()
for idx, mimetype in enumerate(mimetypes):
key = f'mimetype{idx}'
if len(mimetype.split('/')) == 1:
params[key] = f'{mimetype}/%'
else:
params[key] = f'{mimetype}%'
mimetype_wheres.append(f'mimetype LIKE %({key})s')
mimetype_wheres = ' OR '.join(mimetype_wheres)
return f'({mimetype_wheres})', params
wheres.append(f'mimetype LIKE %({key})s')
return f'({" OR ".join(wheres)})', params
return '', params
async def estimate_search(search_str: str, tag_names: List[str], mimetypes: List[str]):
async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str]):
"""
Return FileGroup count of what will be returned if the search is actually performed.
"""
wheres = ['textsearch @@ websearch_to_tsquery(%(search_str)s)']
wheres = []
joins = list()
params = dict(search_str=search_str, tag_names=tag_names, mimetypes=mimetypes)
group_by = ''
having = ''
if search_str:
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
if tag_names:
joins.append('LEFT JOIN tag_file tf on file_group.id = tf.file_group_id')
wheres.append('tf.tag_id IN (select id from tag where name = ANY(%(tag_names)s))')
# Only estimate files that have been tagged with all tag names.
tags_stmt, params_ = tag_names_to_file_group_sub_select(tag_names)
params.update(params_)
wheres.append(f'fg.id = ANY({tags_stmt})')
mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params)
if mimetype_wheres:
wheres.append(mimetype_wheres)
joins = "\n".join(joins)
wheres = "\nAND ".join(wheres)
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
stmt = f'''
SELECT COUNT(*) OVER() AS estimate
FROM file_group
FROM file_group fg
{joins}
WHERE {wheres}
{wheres}
{group_by}
{having}
'''
logger.debug(stmt, params)

View file

@ -29,7 +29,7 @@ from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, APIError, HotspotError, InvalidDownload, \
HotspotPasswordTooShort, NativeOnly, InvalidConfig
from wrolpi.events import get_events, Events
from wrolpi.files.lib import get_file_statistics, estimate_search
from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
truthy_arg
from wrolpi.version import __version__
@ -514,7 +514,7 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ
from modules.archive.lib import search_domains_by_name
file_groups, channels, domains = await asyncio.gather(
estimate_search(body.search_str, body.tag_names, body.mimetypes),
search_file_suggestion_count(body.search_str, body.tag_names, body.mimetypes),
search_channels_by_name(body.search_str),
search_domains_by_name(body.search_str),
)

View file

@ -474,47 +474,74 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
channel_factory(name='Foo')
channel_factory(name='Fool')
channel_factory(name='Bar')
archive_factory(domain='foo.com', contents='contents of foo with bunny')
archive_factory(domain='bar.com', contents='contents of bar')
video_factory(with_caption_file=True)
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
test_session.commit()
body = dict(search_str='foo')
request, response = await test_async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['channels'] == [{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo'},
{'directory': 'Fool',
'id': 2,
'name': 'Fool',
'url': 'https://example.com/Fool'}]
assert response.json['domains'] == [{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}]
assert response.json['file_groups'] == 1
async def assert_results(body: dict, expected_channels=None, expected_domains=None, expected_file_groups=None):
expected_channels = expected_channels or []
expected_domains = expected_domains or []
expected_file_groups = expected_file_groups or 0
body = dict(search_str='bar')
request, response = await test_async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['channels'] == [{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}]
assert response.json['domains'] == [{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}]
assert response.json['file_groups'] == 1
request, response = await test_async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
if expected_channels:
assert response.json['channels'] == expected_channels
assert response.json['domains'] == expected_domains
assert response.json['file_groups'] == expected_file_groups or 0
body = dict(search_str='one')
request, response = await test_async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['channels'] == []
assert response.json['domains'] == []
assert response.json['file_groups'] == 0
await assert_results(
dict(search_str='foo'),
[{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo'},
{'directory': 'Fool',
'id': 2,
'name': 'Fool',
'url': 'https://example.com/Fool'}],
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}],
1,
)
await assert_results(
dict(search_str='bar'),
[{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}],
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}],
1,
)
# "foo" archive and video both contain bunny.
body = dict(search_str='bunny')
request, response = await test_async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == 2
await assert_results(dict(search_str='bunny'), expected_file_groups=2)
# Filtering with mimetypes removes archive result.
body = dict(search_str='bunny', mimetypes=['video'])
request, response = await test_async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == 1
await assert_results(dict(search_str='bunny', mimetypes=['video']), expected_file_groups=1)
# tag1 has been used twice
await assert_results(dict(tag_names=[tag1.name, ]), expected_file_groups=2)
# archive2 was tagged with tag2, but only video contains bunny.
await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), expected_file_groups=1)
# tag2 has been used twice
await assert_results(dict(tag_names=[tag2.name, ]), expected_file_groups=2)
# tag2 has been used twice, but filter by video mimetype.
await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), expected_file_groups=1)
# Can use all filters simultaneously.
await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']),
expected_file_groups=1)
# Only the video has been tagged with both tag1 and tag2.
await assert_results(dict(tag_names=[tag1.name, tag2.name]), expected_file_groups=1)
# tag3 was never used
await assert_results(dict(tag_names=[tag3.name]))
await assert_results(dict(search_str='Does not exist.'))
# No PDFs.
await assert_results(dict(mimetypes=['application/pdf']))
def test_recursive_errors():