From 57d842671f860acb074e51afaa6678e795b41792 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:45:05 -0600 Subject: [PATCH] test: do not leave videos_destination as an invalid template (#656) * test: do not leave videos_destination as an invalid template test_format_videos_destination deliberately sets %(channel)s to prove unknown placeholders fail, then never restored the global WROLPi config. On xdist workers that later ran channel_factory without async_client re-init, the polluted template caused KeyError/FileNotFoundError in test_format_video_filename_uses_channel_name_over_info_json. * test: stop search_suggestions from flaking empty domains on CI A different failure than videos_destination pollution: under xdist load, search_suggestions sometimes returned no domains for a valid query while channels still matched. Harden the path by: - running channel/domain/author/subject searches sequentially on the shared Session (asyncio.gather is not safe with one SQLAlchemy session) - giving archive_factory a URL when only a domain is set so validate() does not clear collection_id via apply_domain() - rewriting test_search_suggestions without hardcoded Collection ids and with unique names that will not collide with other tests --- modules/archive/conftest.py | 4 +++ modules/archive/lib.py | 1 + modules/videos/test/test_lib.py | 44 ++++++++++++++----------- wrolpi/root_api.py | 15 +++++---- wrolpi/test/test_root_api.py | 57 ++++++++++++++++++++------------- 5 files changed, 73 insertions(+), 48 deletions(-) diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py index 2bbf8d1d..42504d80 100644 --- a/modules/archive/conftest.py +++ b/modules/archive/conftest.py @@ -77,6 +77,10 @@ def archive_factory(test_session, archive_directory, make_files_structure, image ) test_session.add(collection) test_session.flush([collection]) + # Archive.validate() -> apply_domain() clears collection_id when URL is missing. + # Give a URL derived from the domain so the link survives validation. + if not url: + url = f'https://{domain}/' else: collection = None diff --git a/modules/archive/lib.py b/modules/archive/lib.py index e1422c09..5ed60c50 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -1493,6 +1493,7 @@ async def search_domains_by_name(session: Session, name: str, limit: int = 5) -> Returns: List of domain dicts matching the search (in old Domain format for backward compatibility) """ + name = name or '' collections = session.query(Collection) \ .filter(Collection.kind == 'domain') \ .filter(Collection.name.ilike(f'%{name}%')) \ diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index b4b1f0d4..84c36c7e 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -337,30 +337,36 @@ def test_link_channel_and_downloads_by_source_id(test_session, channel_factory, async def test_format_videos_destination(async_client, test_directory): """Videos destination is formatted according to the WROLPiConfig.""" wrolpi_config = get_wrolpi_config() + # Restore after mutating the global singleton so later tests (especially those that + # call channel_factory without async_client re-init) do not inherit an invalid template. + original_videos_destination = wrolpi_config.videos_destination - # Channel directory without a tag. - wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_name)s' - assert format_videos_destination('Simple Channel', None, None) \ - == test_directory / 'videos/Simple Channel' + try: + # Channel directory without a tag. + wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_name)s' + assert format_videos_destination('Simple Channel', None, None) \ + == test_directory / 'videos/Simple Channel' - # One tag can be applied to a Channel. - assert format_videos_destination('Simple Channel', 'one', None) \ - == test_directory / 'videos/one/Simple Channel' + # One tag can be applied to a Channel. + assert format_videos_destination('Simple Channel', 'one', None) \ + == test_directory / 'videos/one/Simple Channel' - # Channel Domain is also supported. - wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s' - assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \ - == test_directory / 'videos/one/example.com/Simple Channel' + # Channel Domain is also supported. + wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s' + assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \ + == test_directory / 'videos/one/example.com/Simple Channel' - # Channel name is not required. - wrolpi_config.videos_destination = '%(channel_tag)s/%(channel_domain)s' - assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \ - == test_directory / 'one/example.com' + # Channel name is not required. + wrolpi_config.videos_destination = '%(channel_tag)s/%(channel_domain)s' + assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \ + == test_directory / 'one/example.com' - # Invalid `videos_destination` raises an error. - wrolpi_config.videos_destination = '%(channel_tag)s/%(channel)s' - with pytest.raises(FileNotFoundError): - assert format_videos_destination() + # Invalid `videos_destination` raises an error. + wrolpi_config.videos_destination = '%(channel_tag)s/%(channel)s' + with pytest.raises(FileNotFoundError): + assert format_videos_destination() + finally: + wrolpi_config.videos_destination = original_videos_destination @pytest.mark.asyncio diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 76a1cc0e..c99ae954 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -1,4 +1,3 @@ -import asyncio import json import pathlib import re @@ -774,12 +773,14 @@ async def post_search_suggestions(request: Request, body: schema.SearchSuggestio from modules.docs.lib import search_authors_by_name, search_subjects_by_name session = request.ctx.session - channels, domains, authors, subjects = await asyncio.gather( - search_channels_by_name(session, body.search_str, order_by_video_count=body.order_by_video_count), - search_domains_by_name(session, body.search_str), - search_authors_by_name(session, body.search_str), - search_subjects_by_name(session, body.search_str), - ) + # Run sequentially: a SQLAlchemy Session is not safe for concurrent use, and + # asyncio.gather on the same session has caused intermittent empty results under + # pytest-xdist load even when the coroutines themselves do not await. + channels = await search_channels_by_name( + session, body.search_str, order_by_video_count=body.order_by_video_count) + domains = await search_domains_by_name(session, body.search_str) + authors = await search_authors_by_name(session, body.search_str) + subjects = await search_subjects_by_name(session, body.search_str) ret = dict( channels=channels, diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 8ea33d9d..73cfb9d4 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -632,13 +632,16 @@ async def test_post_vin_number_decoder(async_client): @pytest.mark.asyncio async def test_search_suggestions(test_session, async_client, channel_factory, archive_factory, video_factory): - # WARNING results are cached, this test uses unique queries to avoid conflicts. - channel_factory(name='Foo') - channel_factory(name='Fool') - channel_factory(name='Bar') - archive_factory(domain='foo.com') - archive_factory(domain='bar.com') - video_factory(channel_id=2) # Channel "Fool" will be first in results because it has the most videos. + # Use unique names so this test does not collide with other workers' fixtures or any + # leftover shared-process caches. Do not hardcode DB ids — they shift when other + # fixtures create Channels/Collections first. + channel_foo = channel_factory(name='SugFoo') + channel_fool = channel_factory(name='SugFool') + channel_bar = channel_factory(name='SugBar') + archive_factory(domain='sugfoo.com') + archive_factory(domain='sugbar.com') + # SugFool ranks first for "sugfoo" when ordering by video count. + video_factory(channel_id=channel_fool.id) test_session.commit() async def assert_results(body: dict, expected_channels=None, expected_domains=None): @@ -646,35 +649,45 @@ async def test_search_suggestions(test_session, async_client, channel_factory, a expected_domains = expected_domains or [] request, response = await async_client.post('/api/search_suggestions', json=body) - assert response.status_code == HTTPStatus.OK - if expected_channels: - for channel, expected_channel in zip_longest(response.json['channels'], expected_channels): - assert_dict_contains(channel, expected_channel) - assert response.json['domains'] == expected_domains + assert response.status_code == HTTPStatus.OK, response.json + assert len(response.json['channels']) == len(expected_channels), response.json + for channel, expected_channel in zip_longest(response.json['channels'], expected_channels): + assert_dict_contains(channel, expected_channel) + assert len(response.json['domains']) == len(expected_domains), ( + f"domains mismatch for {body!r}: got {response.json['domains']!r}, " + f"channels={response.json.get('channels')!r}" + ) + for domain, expected_domain in zip_longest(response.json['domains'], expected_domains): + assert_dict_contains(domain, expected_domain) await assert_results( - dict(search_str='foo'), + dict(search_str='sugfoo'), [ - {'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}, - {'directory': 'videos/Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []}, + {'directory': 'videos/SugFool', 'id': channel_fool.id, 'name': 'SugFool', + 'url': 'https://example.com/SugFool', 'downloads': []}, + {'directory': 'videos/SugFoo', 'id': channel_foo.id, 'name': 'SugFoo', + 'url': 'https://example.com/SugFoo', 'downloads': []}, ], - [{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 4}], + [{'directory': 'archive/sugfoo.com', 'domain': 'sugfoo.com'}], ) - # Channel name "Fool" is matched because spaces are stripped in addition to only + # Channel name "SugFool" is matched because spaces are stripped. await assert_results( - dict(search_str='foo l'), + dict(search_str='sugfoo l'), [ - {'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}], + {'directory': 'videos/SugFool', 'id': channel_fool.id, 'name': 'SugFool', + 'url': 'https://example.com/SugFool', 'downloads': []}, + ], [], ) await assert_results( - dict(search_str='bar'), + dict(search_str='sugbar'), [ - {'directory': 'videos/Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []} + {'directory': 'videos/SugBar', 'id': channel_bar.id, 'name': 'SugBar', + 'url': 'https://example.com/SugBar', 'downloads': []}, ], - [{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 5}], + [{'directory': 'archive/sugbar.com', 'domain': 'sugbar.com'}], )