diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 1c0fd05e..cc7a6a13 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -91,8 +91,9 @@ def video_factory(test_session, test_directory): def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file=None, with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False, - source_id: str = None) -> Video: + source_id: str = None, tag_names: List[str] = None) -> Video: title = title or str(uuid4()) + tag_names = tag_names or list() if with_video_file and isinstance(with_video_file, (pathlib.Path, str)): # Put the video exactly where specified for the test. @@ -134,6 +135,10 @@ def video_factory(test_session, test_directory): video.source_id = source_id or title video.file_group.published_datetime = upload_date video.validate() + + for tag_name in tag_names: + video.add_tag(tag_name) + return video return factory diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 5b0607b0..0d6d5f67 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -46,7 +46,8 @@ except ImportError: logger = logger.getChild(__name__) __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files', - 'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', 'estimate_search', + 'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', + 'search_file_suggestion_count', 'move', 'rename', 'delete_directory', 'handle_file_group_search_results'] @@ -709,7 +710,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] wheres.append('model = %(model)s') if tags: - # Filter all FileGroups by those that have been tagged with the provided tag names. + # Filter all FileGroups by those that have been tagged with all the provided tag names. tags_stmt, params_ = tag_names_to_file_group_sub_select(tags) params.update(params_) wheres.append(f'fg.id = ANY({tags_stmt})') @@ -1054,42 +1055,49 @@ def get_refresh_progress() -> RefreshProgress: def mimetypes_to_sql_wheres(mimetypes: List[str], params: dict): if mimetypes: - mimetype_wheres = [] + wheres = list() for idx, mimetype in enumerate(mimetypes): key = f'mimetype{idx}' if len(mimetype.split('/')) == 1: params[key] = f'{mimetype}/%' else: params[key] = f'{mimetype}%' - mimetype_wheres.append(f'mimetype LIKE %({key})s') - mimetype_wheres = ' OR '.join(mimetype_wheres) - return f'({mimetype_wheres})', params + wheres.append(f'mimetype LIKE %({key})s') + return f'({" OR ".join(wheres)})', params return '', params -async def estimate_search(search_str: str, tag_names: List[str], mimetypes: List[str]): +async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str]): """ Return FileGroup count of what will be returned if the search is actually performed. """ - wheres = ['textsearch @@ websearch_to_tsquery(%(search_str)s)'] + wheres = [] joins = list() params = dict(search_str=search_str, tag_names=tag_names, mimetypes=mimetypes) + group_by = '' + having = '' + if search_str: + wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)') if tag_names: - joins.append('LEFT JOIN tag_file tf on file_group.id = tf.file_group_id') - wheres.append('tf.tag_id IN (select id from tag where name = ANY(%(tag_names)s))') + # Only estimate files that have been tagged with all tag names. + tags_stmt, params_ = tag_names_to_file_group_sub_select(tag_names) + params.update(params_) + wheres.append(f'fg.id = ANY({tags_stmt})') mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params) if mimetype_wheres: wheres.append(mimetype_wheres) joins = "\n".join(joins) - wheres = "\nAND ".join(wheres) + wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else '' stmt = f''' SELECT COUNT(*) OVER() AS estimate - FROM file_group + FROM file_group fg {joins} - WHERE {wheres} + {wheres} + {group_by} + {having} ''' logger.debug(stmt, params) diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index bdd951fe..e1677694 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -29,7 +29,7 @@ from wrolpi.downloader import download_manager from wrolpi.errors import WROLModeEnabled, APIError, HotspotError, InvalidDownload, \ HotspotPasswordTooShort, NativeOnly, InvalidConfig from wrolpi.events import get_events, Events -from wrolpi.files.lib import get_file_statistics, estimate_search +from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \ truthy_arg from wrolpi.version import __version__ @@ -514,7 +514,7 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ from modules.archive.lib import search_domains_by_name file_groups, channels, domains = await asyncio.gather( - estimate_search(body.search_str, body.tag_names, body.mimetypes), + search_file_suggestion_count(body.search_str, body.tag_names, body.mimetypes), search_channels_by_name(body.search_str), search_domains_by_name(body.search_str), ) diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 716350f3..b196d6f9 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -474,47 +474,74 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto channel_factory(name='Foo') channel_factory(name='Fool') channel_factory(name='Bar') - archive_factory(domain='foo.com', contents='contents of foo with bunny') - archive_factory(domain='bar.com', contents='contents of bar') - video_factory(with_caption_file=True) + tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory() + archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ]) + archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ]) + video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name]) test_session.commit() - body = dict(search_str='foo') - request, response = await test_async_client.post('/api/search_suggestions', json=body) - assert response.status_code == HTTPStatus.OK - assert response.json['channels'] == [{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo'}, - {'directory': 'Fool', - 'id': 2, - 'name': 'Fool', - 'url': 'https://example.com/Fool'}] - assert response.json['domains'] == [{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}] - assert response.json['file_groups'] == 1 + async def assert_results(body: dict, expected_channels=None, expected_domains=None, expected_file_groups=None): + expected_channels = expected_channels or [] + expected_domains = expected_domains or [] + expected_file_groups = expected_file_groups or 0 - body = dict(search_str='bar') - request, response = await test_async_client.post('/api/search_suggestions', json=body) - assert response.status_code == HTTPStatus.OK - assert response.json['channels'] == [{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}] - assert response.json['domains'] == [{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}] - assert response.json['file_groups'] == 1 + request, response = await test_async_client.post('/api/search_suggestions', json=body) + assert response.status_code == HTTPStatus.OK + if expected_channels: + assert response.json['channels'] == expected_channels + assert response.json['domains'] == expected_domains + assert response.json['file_groups'] == expected_file_groups or 0 - body = dict(search_str='one') - request, response = await test_async_client.post('/api/search_suggestions', json=body) - assert response.status_code == HTTPStatus.OK - assert response.json['channels'] == [] - assert response.json['domains'] == [] - assert response.json['file_groups'] == 0 + await assert_results( + dict(search_str='foo'), + [{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo'}, + {'directory': 'Fool', + 'id': 2, + 'name': 'Fool', + 'url': 'https://example.com/Fool'}], + [{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}], + 1, + ) + + await assert_results( + dict(search_str='bar'), + [{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}], + [{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}], + 1, + ) # "foo" archive and video both contain bunny. - body = dict(search_str='bunny') - request, response = await test_async_client.post('/api/search_suggestions', json=body) - assert response.status_code == HTTPStatus.OK - assert response.json['file_groups'] == 2 + await assert_results(dict(search_str='bunny'), expected_file_groups=2) # Filtering with mimetypes removes archive result. - body = dict(search_str='bunny', mimetypes=['video']) - request, response = await test_async_client.post('/api/search_suggestions', json=body) - assert response.status_code == HTTPStatus.OK - assert response.json['file_groups'] == 1 + await assert_results(dict(search_str='bunny', mimetypes=['video']), expected_file_groups=1) + + # tag1 has been used twice + await assert_results(dict(tag_names=[tag1.name, ]), expected_file_groups=2) + + # archive2 was tagged with tag2, but only video contains bunny. + await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), expected_file_groups=1) + + # tag2 has been used twice + await assert_results(dict(tag_names=[tag2.name, ]), expected_file_groups=2) + + # tag2 has been used twice, but filter by video mimetype. + await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), expected_file_groups=1) + + # Can use all filters simultaneously. + await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']), + expected_file_groups=1) + + # Only the video has been tagged with both tag1 and tag2. + await assert_results(dict(tag_names=[tag1.name, tag2.name]), expected_file_groups=1) + + # tag3 was never used + await assert_results(dict(tag_names=[tag3.name])) + + await assert_results(dict(search_str='Does not exist.')) + + # No PDFs. + await assert_results(dict(mimetypes=['application/pdf'])) def test_recursive_errors():