wrolpi/modules/videos/test/test_downloader.py

import asyncio
import json
import shutil
from copy import copy
from http import HTTPStatus
from unittest import mock

import pytest

from modules.videos.downloader import VideoDownloader, \
    get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
    prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download, \
    parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url, format_selector, VIDEO_RESOLUTION_MAP
from modules.videos.lib import get_videos_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
from wrolpi.downloader import Download, DownloadResult, get_download_manager_config
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
from wrolpi.test.common import skip_ci
from wrolpi.vars import PROJECT_DIR

example_video_json = {
    'age_limit': 0,
    'average_rating': 4.6923075,
    'categories': ['Howto & Style'],
    'channel': 'channel name',
    'channel_id': 'channel id',
    'channel_url': 'https://example.com/example_video_json_channel',
    'duration': 635,
    'extractor': 'youtube',
    'extractor_key': 'Youtube',
    'id': 'some long id',
    'is_live': None,
    'like_count': 24,
    'subtitles': {},
    'title': "The video's title",
    'upload_date': '20190707',
    'uploader': 'uploader name',
    'uploader_id': 'uploader id',
    'uploader_url': 'https://example.com/example_video_json_uploader',
    'view_count': 406,
    'webpage_url': 'webpage url',
    'webpage_url_basename': 'watch',
}

example_channel_json = {
    '_type': 'playlist',
    'channel_id': 'some id',
    'entries': [
        {'_type': 'url',
         'description': None,
         'duration': None,
         'id': 'video 1 id',
         'ie_key': 'Youtube',
         'title': 'video 1 title',
         'uploader': None,
         'url': 'video_1_url',
         'view_count': 58504,
         'upload_date': '20190707',
         'webpage_url': 'https://youtube.com/watch?v=video_1_url'},
        {'_type': 'url',
         'description': None,
         'duration': None,
         'id': 'video 2 id',
         'ie_key': 'Youtube',
         'title': 'video 2 title',
         'uploader': None,
         'url': 'video_2_url',
         'view_count': 1413,
         'upload_date': '20190707',
         'webpage_url': 'https://youtube.com/watch?v=video_2_url'},
    ],
    'extractor': 'youtube:tab',
    'extractor_key': 'YoutubeTab',
    'id': 'some id',
    'title': 'channel title',
    'uploader': 'the uploader',
    'webpage_url': 'channel url',
    'webpage_url_basename': 'videos',
}


@pytest.mark.asyncio
async def test_download_no_channel(test_session, video_download_manager, test_directory, await_switches,
                                   mock_video_extract_info, mock_video_process_runner):
    """A video can be downloaded even if it does not have a Channel."""
    channel_dir = test_directory / 'NO CHANNEL'
    channel_dir.mkdir(parents=True)
    video_path = channel_dir / 'video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)

    channel_downloader, video_downloader = video_download_manager.instances

    # Video has no channel
    info_json = copy(example_video_json)
    del info_json['channel']
    del info_json['channel_id']
    del info_json['channel_url']

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
    mock_video_extract_info.return_value = info_json
    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
        video_download_manager.create_download(test_session, url, video_downloader.name)
        await video_download_manager.wait_for_all_downloads()

    mock_video_process_runner.assert_called_once()

    video: Video = test_session.query(Video).one()
    assert str(video.video_path) == f'{channel_dir}/video.mp4'


@pytest.mark.asyncio
async def test_download_video_tags(test_session, video_download_manager, video_file_factory, test_directory,
                                   mock_video_extract_info, mock_video_process_runner, tag_factory, await_switches):
    """A Video is tagged when Download record requires it."""
    video_path = video_file_factory()
    tag1, tag2 = await tag_factory(), await tag_factory()

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
    mock_video_extract_info.return_value = copy(example_video_json)
    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
        settings = dict(tag_names=[tag1.name, tag2.name])
        video_download_manager.create_download(test_session, url, video_downloader.name, settings=settings)
        await video_download_manager.wait_for_all_downloads()

    mock_video_process_runner.assert_called_once()

    video: Video = test_session.query(Video).one()
    assert [i.tag.name for i in video.file_group.tag_files] == [tag1.name, tag2.name]


@skip_ci
@pytest.mark.asyncio
async def test_download_channel(test_session, test_directory, simple_channel, video_download_manager, video_file,
                                mock_video_extract_info, mock_video_prepare_filename, await_switches,
                                mock_video_process_runner):
    """Downloading (updating the catalog of) a Channel updates its info_json.

    If a Channel has `match_regex` only those videos with matching titles will be downloaded."""
    url = 'https://www.youtube.com/c/LearningSelfReliance/videos'

    mock_video_prepare_filename.return_value = video_file
    mock_video_extract_info.return_value = example_channel_json
    with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
        mock_get_channel.return_value = simple_channel
        simple_channel.get_or_create_download(test_session, url, 60)
        # Download channel.
        await video_download_manager.wait_for_all_downloads()
        # Download videos.
        await video_download_manager.wait_for_all_downloads()

    def reset_downloads():
        [i.delete(add_to_skip_list=False) for i in test_session.query(Download)]

    # Let background tasks run.
    await asyncio.sleep(0)

    # Two videos are in the example channel.
    downloads = video_download_manager.get_once_downloads(test_session)
    downloads = filter(lambda i: 'watch' in i.url, downloads)
    assert {i.url for i in downloads} == \
           {'https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url'}
    assert all(i.status_code == 'complete' for i in downloads)

    reset_downloads()

    # A channel with `title_include` only returns matching video URLs.
    test_session.commit()
    with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
        mock_get_channel.return_value = simple_channel
        download = simple_channel.get_or_create_download(test_session, url, 60)
        download.frequency = 100
        download.settings = {'title_include': '2'}
        await video_download_manager.wait_for_all_downloads()
    downloads = video_download_manager.get_once_downloads(test_session)
    assert {i.url for i in downloads} == {'https://youtube.com/watch?v=video_2_url'}
    assert downloads[0].settings == {
        'collection_id': simple_channel.collection_id,
        'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
        'destination': str(test_directory),
        'parent_download_url': url,
    }

    reset_downloads()

    # A channel with `title_exclude` only returns non-matching video URLs.
    test_session.commit()
    with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
        mock_get_channel.return_value = simple_channel
        download = simple_channel.get_or_create_download(test_session, url, 60)
        download.frequency = 100
        download.settings = {'title_exclude': '2'}
        await video_download_manager.wait_for_all_downloads()
    downloads = video_download_manager.get_once_downloads(test_session)
    assert {i.url for i in downloads} == {'https://youtube.com/watch?v=video_1_url'}
    assert downloads[0].settings == {
        'collection_id': simple_channel.collection_id,
        'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
        'destination': str(test_directory),
        'parent_download_url': url,
    }


@pytest.mark.asyncio
async def test_get_or_create_channel(async_client, test_session, test_directory, tag_factory, await_switches):
    """A Channel may need to be created for an arbitrary download.

    Attempt to use an existing Channel if we can match it.
    """
    from wrolpi.collections import Collection
    one, two = await tag_factory(), await tag_factory()

    # Create Collections for each Channel
    def create_channel_with_collection(name, source_id=None, url=None):
        collection = Collection(name=name, kind='channel')
        test_session.add(collection)
        test_session.flush([collection])
        channel = Channel(collection_id=collection.id, source_id=source_id, url=url)
        test_session.add(channel)
        test_session.flush([channel])
        return channel

    c1 = create_channel_with_collection('foo', source_id='foo', url='https://example.com')
    c2 = create_channel_with_collection('bar', source_id='bar')
    c3 = create_channel_with_collection('baz', source_id='baz', url='https://example.net')
    c4 = create_channel_with_collection('qux')
    test_session.commit()

    # All existing channels should be used.
    tests = [
        (dict(source_id='foo'), c1),
        (dict(url='https://example.com'), c1),
        (dict(url='https://example.com', source_id='bar'), c2),  # source_id has priority.
        (dict(name='foo', source_id='bar'), c2),
        (dict(source_id='bar'), c2),
        (dict(source_id='baz'), c3),
        (dict(name='qux'), c4),
    ]
    for kwargs, expected in tests:
        channel = get_or_create_channel(test_session, **kwargs)
        assert expected.id == channel.id, f'Expected {expected} for {kwargs} but got {channel}'

    # A new channel is created.  It will not be automatically downloaded.
    channel = get_or_create_channel(test_session, source_id='quux', name='quux', url='https://example.org')
    assert channel.id == 5
    assert channel.source_id == 'quux'
    assert channel.name == 'quux'
    assert channel.url == 'https://example.org'

    # New channel can be retrieved.
    assert get_or_create_channel(test_session, source_id='quux') == channel

    # A Channel can be created with a tag name.
    channel = get_or_create_channel(test_session, name='One Channel', tag_name=one.name)
    await await_switches()
    assert channel.directory == test_directory / f'videos/one/One Channel'
    assert channel.tag_name == one.name and channel.tag == one

    # Getting the same Channel will not change the tag.
    channel = get_or_create_channel(test_session, name='One Channel', tag_name=two.name)
    assert channel.directory == test_directory / f'videos/one/One Channel'
    assert channel.tag_name == one.name and channel.tag == one
    channel = get_or_create_channel(test_session, name='One Channel')
    assert channel.directory == test_directory / f'videos/one/One Channel'
    assert channel.directory.is_dir()
    assert channel.tag_name == one.name and channel.tag == one


@pytest.mark.asyncio
async def test_get_or_create_channel_conflict_recovery(async_client, test_session, test_directory):
    """get_or_create_channel should find an existing channel even when source_id and url don't match,
    rather than raising ChannelNameConflict."""
    from wrolpi.collections import Collection

    # Create a channel with a specific source_id and url.
    collection = Collection(name='My Channel', kind='channel')
    test_session.add(collection)
    test_session.flush([collection])
    channel = Channel(collection_id=collection.id, source_id='old_source_id',
                      url='https://www.youtube.com/channel/UCold')
    test_session.add(channel)
    test_session.commit()

    # Calling with different source_id and url but same name should return the existing channel.
    result = get_or_create_channel(test_session, source_id='new_source_id',
                                   url='https://www.youtube.com/@mychannel', name='My Channel')
    assert result.id == channel.id


@pytest.mark.asyncio
async def test_get_or_create_channel_youtube_channel_url(async_client, test_session, test_directory):
    """get_or_create_channel should find an existing channel when the URL contains a YouTube /channel/ ID
    that matches the channel's source_id."""
    from wrolpi.collections import Collection

    collection = Collection(name='WROLPi', kind='channel')
    test_session.add(collection)
    test_session.flush([collection])
    channel = Channel(collection_id=collection.id, source_id='UC4t8bw1besFTyjW7ZBCOIrw',
                      url='https://www.youtube.com/@wrolpi')
    test_session.add(channel)
    test_session.commit()

    # A /channel/UCXXX URL should match via source_id extraction.
    result = get_or_create_channel(test_session,
                                   url='https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw',
                                   name='WROLPi Channel')
    assert result.id == channel.id


@pytest.mark.asyncio
async def test_get_or_create_channel_with_destination(async_client, test_session, test_directory):
    """A new Channel should be created at the destination directory when provided."""
    destination = test_directory / 'custom/my_channel'

    channel = get_or_create_channel(test_session, source_id='abc', name='My Channel',
                                    url='https://example.com', destination=destination)
    assert channel.name == 'My Channel'
    assert channel.directory == destination
    assert destination.is_dir()


@pytest.mark.asyncio
async def test_get_or_create_channel_destination_conflict(async_client, test_session, test_directory):
    """When destination matches an existing channel's directory, that channel should be returned
    instead of trying to create a new one (which would raise ChannelDirectoryConflict)."""
    from wrolpi.collections import Collection

    # Create an existing channel at the destination directory.
    existing_dir = test_directory / 'videos/existing'
    existing_dir.mkdir(parents=True)
    collection = Collection(name='Existing Channel', kind='channel', directory=existing_dir)
    test_session.add(collection)
    test_session.flush([collection])
    channel = Channel(collection_id=collection.id, source_id='existing')
    test_session.add(channel)
    test_session.commit()

    # Destination matches an existing channel, so it should be returned even though source_id/name/url don't match.
    result = get_or_create_channel(test_session, source_id='new_id', name='New Channel',
                                   url='https://example.com', destination=existing_dir)
    assert result.id == channel.id


@pytest.mark.asyncio
async def test_get_or_create_channel_destination_nested_conflict(async_client, test_session, test_directory):
    """When destination is nested inside an existing channel's directory, the parent channel should be returned."""
    from wrolpi.collections import Collection

    parent_dir = test_directory / 'videos/parent_channel'
    parent_dir.mkdir(parents=True)
    collection = Collection(name='Parent Channel', kind='channel', directory=parent_dir)
    test_session.add(collection)
    test_session.flush([collection])
    channel = Channel(collection_id=collection.id, source_id='parent')
    test_session.add(channel)
    test_session.commit()

    # Nested directory should find the parent channel.
    nested_dir = parent_dir / 'nested'
    result = get_or_create_channel(test_session, source_id='nested_id', name='Nested Channel',
                                   url='https://example.net', destination=nested_dir)
    assert result.id == channel.id


@pytest.mark.asyncio
async def test_get_or_create_channel_computed_directory_conflict(async_client, test_session, test_directory):
    """When no existing channel matches by source_id/url/name, but the directory that would be created
    already belongs to a channel, that channel should be returned instead of raising.

    YouTube collaboration videos report the primary uploader's channel, which may not match any local
    channel by name (e.g. the local channel was renamed) even though its directory is already taken."""
    from wrolpi.collections import Collection

    existing_dir = test_directory / 'videos/Uploader'
    existing_dir.mkdir(parents=True)
    collection = Collection(name='Uploader (old name)', kind='channel', directory=existing_dir)
    test_session.add(collection)
    test_session.flush([collection])
    channel = Channel(collection_id=collection.id, source_id='UCold', url='https://www.youtube.com/@olduploader')
    test_session.add(channel)
    test_session.commit()

    result = get_or_create_channel(test_session, source_id='UCnew',
                                   url='https://www.youtube.com/channel/UCnew', name='Uploader')
    assert result.id == channel.id


@pytest.mark.asyncio
async def test_get_or_create_channel_existing_ignores_destination(async_client, test_session, test_directory):
    """When a channel already exists (matched by source_id), the destination parameter should be ignored."""
    from wrolpi.collections import Collection

    existing_dir = test_directory / 'videos/existing'
    existing_dir.mkdir(parents=True)
    collection = Collection(name='Existing', kind='channel', directory=existing_dir)
    test_session.add(collection)
    test_session.flush([collection])
    channel = Channel(collection_id=collection.id, source_id='match_me', url='https://example.com')
    test_session.add(channel)
    test_session.commit()

    # Even with a different destination, the existing channel is returned.
    other_dir = test_directory / 'videos/other'
    result = get_or_create_channel(test_session, source_id='match_me', name='Existing',
                                   destination=other_dir)
    assert result.id == channel.id
    assert result.directory == existing_dir


@pytest.mark.asyncio
async def test_get_channel_prefers_destination_over_ytdlp(async_client, test_session, test_directory, channel_factory):
    """When yt-dlp metadata identifies ChannelA but the download destination is ChannelB's directory,
    _get_channel should return ChannelB (the destination's channel)."""
    channel_a = channel_factory(source_id='channel_a_id', name='ChannelA',
                                url='https://example.com/channel_a')
    channel_b = channel_factory(source_id='channel_b_id', name='ChannelB',
                                url='https://example.com/channel_b')

    # Download has yt-dlp metadata matching ChannelA, but destination is ChannelB's directory.
    download = Download(
        url='https://example.com/video1',
        destination=channel_b.directory,
        info_json={'channel': 'ChannelA', 'channel_id': 'channel_a_id',
                   'channel_url': 'https://example.com/channel_a'},
    )

    channel = await VideoDownloader._get_channel(test_session, download)

    assert channel.id == channel_b.id, 'Should prefer the destination channel over yt-dlp metadata'


@pytest.mark.asyncio
async def test_get_channel_uses_ytdlp_when_destination_matches(async_client, test_session, test_directory,
                                                                channel_factory):
    """When yt-dlp metadata identifies ChannelA and the download destination is also ChannelA's directory,
    _get_channel should return ChannelA (no conflict)."""
    channel_a = channel_factory(source_id='channel_a_id', name='ChannelA',
                                url='https://example.com/channel_a')

    download = Download(
        url='https://example.com/video1',
        destination=channel_a.directory,
        info_json={'channel': 'ChannelA', 'channel_id': 'channel_a_id',
                   'channel_url': 'https://example.com/channel_a'},
    )

    channel = await VideoDownloader._get_channel(test_session, download)

    assert channel.id == channel_a.id


def test_channel_downloader_hidden(video_download_manager):
    """
    ChannelDownloader should not be presented to the User.
    """
    downloaders = video_download_manager.list_downloaders()
    assert [i.__json__() for i in downloaders] == [(dict(name='video', pretty_name='Videos')), ]


def test_bad_downloader(test_session, video_download_manager):
    """
    Attempting to use an unknown downloader should raise an error.
    """
    with pytest.raises(InvalidDownload):
        video_download_manager.create_download(test_session, 'https://example.com', downloader_name='bad downloader')


@pytest.mark.parametrize('url,expected', [
    ('https://www.youtube.com/feeds/videos.xml?channel_id=UC2JGH5ZBtWQJuTYCyfI3M3Q', True),
    ('http://youtube.com/feeds/videos.xml?channel_id=UC123', True),
    ('https://www.youtube.com/feeds/videos.xml', True),
    ('https://www.youtube.com/@SomeChannel/videos', False),
    ('https://www.youtube.com/watch?v=abc123', False),
    ('https://example.com/feeds/videos.xml', False),
    ('', False),
    (None, False),
])
def test_is_youtube_rss_feed_url(url, expected):
    """Detect YouTube RSS feed URLs so ChannelDownloader can refuse to process them."""
    assert is_youtube_rss_feed_url(url) is expected


@pytest.mark.asyncio
async def test_channel_downloader_rejects_rss_feed_url(test_session, mock_video_extract_info):
    """ChannelDownloader must refuse YouTube RSS feed URLs.

    Previously, yt-dlp would return no `uploader`/`channel_id` for an RSS feed, so
    `name = info.get('uploader') or info.get('webpage_url_basename')` fell back to
    'videos.xml' and `source_id` fell back to 'videos', creating a junk channel named
    'videos.xml' in directory 'videos/videos.xml' that never received any videos.
    """
    download = Download(
        url='https://www.youtube.com/feeds/videos.xml?channel_id=UC2JGH5ZBtWQJuTYCyfI3M3Q',
        downloader=channel_downloader.name,
    )
    test_session.add(download)
    test_session.commit()

    # If the rejection was missing, ChannelDownloader would call extract_info and create a channel.
    mock_video_extract_info.return_value = {
        'id': 'videos',
        'webpage_url_basename': 'videos.xml',
        'entries': [],
    }

    with pytest.raises(UnrecoverableDownloadError):
        # RSS-feed rejection happens in the prepare phase (sync), so we don't even
        # need to drive execute_download here.
        channel_downloader.prepare_download(test_session, download)

    # No Channel should have been created from the RSS URL.
    assert test_session.query(Channel).count() == 0
    mock_video_extract_info.assert_not_called()


@pytest.mark.asyncio
async def test_video_download(test_session, test_directory, mock_video_extract_info, simple_channel, await_switches,
                              video_download_manager, mock_video_process_runner, image_file):
    """A video download is performed, files are grouped."""
    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)

    video_path = simple_channel.directory / 'a video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
    # Create a poster file which was downloaded.
    poster_path = video_path.with_suffix('.png')
    image_file.rename(poster_path)
    # `formats` should be cleaned from info json.
    info_json_path = video_path.with_suffix('.info.json')
    info_json_path.write_text(json.dumps({'duration': 5, 'formats': [1, 2, 3]}))
    # .part file should be deleted
    part_path = video_path.with_suffix('.mp4.part')
    part_path.touch()

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})

        video_download_manager.create_download(test_session, url, video_downloader.name)
        await video_download_manager.wait_for_all_downloads()

        mock_video_process_runner.assert_called_once()
        download, cmd, out_dir = mock_video_process_runner.call_args[0]

    download: Download = test_session.query(Download).one()
    assert download.url in cmd
    assert test_session.query(Channel).one()

    video: Video = test_session.query(Video).one()
    assert video.video_path.is_absolute() and video.video_path == video_path, 'Video path is not absolute'
    assert video.video_path.is_absolute() and video.poster_path == poster_path, 'Video poster was not discovered'
    assert not any([i for i in video.file_group.my_paths() if i.name.endswith('.part')]), \
        'Part file should not be in group'
    assert not part_path.exists(), 'Part file should be deleted.'
    # Some info json data is deleted during download.
    assert video.get_info_json(), 'Download should keep info json file'
    assert 'duration' in video.get_info_json(), 'Duration should stay in info json'
    assert 'formats' not in video.get_info_json(), 'formats should be deleted from info json'


@pytest.mark.asyncio
async def test_video_download_deletes_format_leftovers(test_session, test_directory, mock_video_extract_info,
                                                       simple_channel, await_switches, video_download_manager,
                                                       mock_video_process_runner, image_file):
    """yt-dlp's per-format files (`<stem>.f313.webm`) left by an earlier, unmerged download are deleted, whatever
    their extension, so they do not join the new Video's FileGroup."""
    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)

    video_path = simple_channel.directory / 'a video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
    # An earlier download's video-only format file was never merged; WROLPi indexed it and made a poster for it.
    leftover_video = simple_channel.directory / 'a video.f313.webm'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', leftover_video)
    leftover_poster = simple_channel.directory / 'a video.f313.jpg'
    image_file.rename(leftover_poster)
    leftover_ytdl = simple_channel.directory / 'a video.f251.webm.ytdl'
    leftover_ytdl.write_text('{}')
    # Not a format file of this video.
    other = simple_channel.directory / 'a video.f313 notes.txt'
    other.write_text('keep me')

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})

        video_download_manager.create_download(test_session, url, video_downloader.name)
        await video_download_manager.wait_for_all_downloads()

    video: Video = test_session.query(Video).one()
    assert video.video_path == video_path
    assert not leftover_video.exists() and not leftover_poster.exists() and not leftover_ytdl.exists()
    assert not any('.f313.' in i.name or '.f251.' in i.name for i in video.file_group.my_paths())
    assert other.exists()


@pytest.mark.asyncio
async def test_download_info_json_finds_existing_video(test_session, test_directory, video_factory):
    """`download_info_json` looks up the existing Video by its URL.

    Regression: `Video.get_by_url` was called with its arguments swapped — `(url, session)` instead of
    `(session, url)` — which raised `AttributeError: 'str' object has no attribute 'query'`."""
    source_id = 'video-source-id'
    url = 'https://example.com/the-video'
    video = video_factory(with_info_json={'id': source_id}, source_id=source_id)
    video.file_group.url = url
    test_session.commit()

    download = Download(url=url, info_json={'id': source_id}, settings={})

    # The info_json download itself fails here, but the buggy existing-video lookup runs first.
    with mock.patch('modules.videos.downloader.download_video_info_json') as mock_download:
        mock_download.side_effect = Exception('no network')
        result = await VideoDownloader.download_info_json(download, video.video_path)

    assert result.success is False, 'Download should fail gracefully once the info json download fails'
    mock_download.assert_called_once_with(url)


@pytest.mark.asyncio
async def test_download_result(test_session, test_directory, video_download_manager, mock_video_process_runner,
                               mock_video_extract_info, image_file, await_switches):
    """VideoDownloader returns a DownloadResult when complete."""
    channel_directory = test_directory / 'videos/channel name'
    channel_directory.mkdir(parents=True)
    video_file = channel_directory / 'the video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file)
    image_file.rename(video_file.with_suffix('.jpg'))

    mock_video_extract_info.return_value = example_video_json
    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_prepare_filename.return_value = [video_file, {'id': 'foo'}]
        video_download_manager.create_download(test_session, 'https://example.com', video_downloader.name)
        await video_download_manager.wait_for_all_downloads()
        # Sleep to allow background tasks to finish.
        await asyncio.sleep(1)

    download: Download = test_session.query(Download).one()
    assert download.url == 'https://example.com'
    video = test_session.query(Video).one()
    assert download.location == f'/videos/{video.file_group_id}'

    # Video has its files.
    video = test_session.query(Video).one()
    assert video.video_path.is_file(), 'Video file was not found.'
    assert video.poster_path.is_file(), 'Poster file was not found.'


@pytest.mark.asyncio
async def test_download_destination(test_session, test_directory, video_download_manager, await_switches,
                                    mock_video_process_runner, mock_video_extract_info, mock_video_prepare_filename):
    """A Video can be downloaded to a directory other than it's Channel's directory."""
    # yt-dlp would return the video json.
    mock_video_extract_info.return_value = example_video_json
    # This result will be ignored during the test, we just need some value so the download does not fail.
    mock_video_prepare_filename.return_value = str(test_directory / 'test video.mp4')

    video_download_manager.create_download(test_session, 'https://example.com/1', downloader_name=VideoDownloader.name)
    await video_download_manager.wait_for_all_downloads()
    # Output directory matches the channel directory.
    assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \
        .startswith(f'{test_directory}/videos/channel name/%(uploader)s')

    mock_video_prepare_filename.reset_mock()

    video_download_manager.create_download(test_session, 'https://example.com/2', downloader_name=VideoDownloader.name,
                                           destination=f'{test_directory}/custom')
    await video_download_manager.wait_for_all_downloads()
    # Output directory matches the custom directory specified.
    assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \
        .startswith(f'{test_directory}/custom/%(uploader)s')


example_playlist_json = {
    '_type': 'playlist',
    'availability': None,
    'channel': 'the channel name',
    'channel_follower_count': None,
    'channel_id': 'the channel id',
    'channel_url': 'channel url',
    'description': '',
    'entries': [
        {'_type': 'url',
         'description': None,
         'duration': None,
         'id': 'video 2 id',
         'ie_key': 'Youtube',
         'title': 'video 2 title',
         'uploader': None,
         'upload_date': '20241108',
         'url': 'https://www.youtube.com/shorts/video_2_url',
         'view_count': 1413,
         'webpage_url': 'https://www.youtube.com/shorts/video_2_url'},
        {'_type': 'url',
         'description': None,
         'duration': None,
         'id': 'video 1 id',
         'ie_key': 'Youtube',
         'title': 'video 1 title',
         'uploader': None,
         'upload_date': '20241108',
         'url': 'https://www.youtube.com/watch?v=video_1_url',
         'view_count': 58504,
         'webpage_url': 'https://www.youtube.com/watch?v=video_1_url'},
        {'_type': 'url',
         'description': None,
         'duration': None,
         'id': 'video 3 id',
         'ie_key': 'Youtube',
         'title': 'video 3 title',
         'uploader': None,
         'upload_date': '20241108',
         'url': 'https://youtube.com/watch?v=video_3_url',
         'view_count': 58504,
         'webpage_url': 'https://youtube.com/watch?v=video_3_url'},
    ],
    'extractor': 'youtube:tab',
    'extractor_key': 'YoutubeTab',
    'id': 'the playlist id',
    'modified_date': '20220426',
    'original_url': 'original url',
    'playlist_count': 10,
    'requested_entries': None,
    'tags': [],
    'title': 'some title',
    'uploader': 'Playlist Uploader',
    'uploader_id': 'uploader id',
    'uploader_url': 'uploader url',
    'view_count': 22298,
    'webpage_url': 'webpage url',
    'webpage_url_basename': 'playlist',
    'webpage_url_domain': 'youtube.com',
}


@pytest.mark.asyncio
async def test_download_playlist(test_session, test_directory, mock_video_extract_info, video_download_manager,
                                 video_file):
    """All videos in a playlist can be downloaded for it's Channel."""
    download = Download(url='https://example.com/playlist-url')
    test_session.add(download)
    channel = get_or_create_channel(test_session, source_id=example_playlist_json['channel_id'], url=download.url,
                                    name=example_channel_json['uploader'])
    video = Video.from_paths(test_session, video_file)
    video.channel_id = channel.id
    video.source_id = 'video 1 id'
    video.file_group.url = 'https://www.youtube.com/watch?v=video_1_url'
    test_session.commit()

    mock_video_extract_info.return_value = example_playlist_json  # Playlist info is fetched first.

    # Drive the phase-split API directly: prepare → execute → finalize.  No dispatch
    # loop, so child video downloads are never spawned here — no need to stub them.
    from modules.archive.conftest import make_test_ctx
    prepared = channel_downloader.prepare_download(test_session, download)
    executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
    result = channel_downloader.finalize_download(test_session, download, executed)
    assert result.success is True, 'Download was not successful'
    assert set(result.downloads) == {
        'https://www.youtube.com/watch?v=video_2_url',  # Shorts is converted to regular URL.
        'https://youtube.com/watch?v=video_3_url',
    }


def test_get_missing_videos_views_order(test_session):
    """The 'views' download order sorts videos most-viewed-first.  Entries missing a `view_count`
    (some listings omit it) sort last rather than raising a KeyError."""
    from modules.videos.downloader import channel_downloader
    download = Download(
        url='https://www.youtube.com/@example/videos',
        settings={'download_order': 'views'},
        info_json={'entries': [
            {'_type': 'url', 'id': 'a', 'title': 'a', 'url': 'https://youtube.com/watch?v=a', 'view_count': 918},
            {'_type': 'url', 'id': 'b', 'title': 'b', 'url': 'https://youtube.com/watch?v=b'},  # No view_count.
            {'_type': 'url', 'id': 'c', 'title': 'c', 'url': 'https://youtube.com/watch?v=c', 'view_count': 3000},
        ]},
    )
    test_session.add(download)
    test_session.commit()

    missing_videos = channel_downloader.get_missing_videos(test_session, download)
    assert missing_videos == [
        'https://youtube.com/watch?v=c',
        'https://youtube.com/watch?v=a',
        'https://youtube.com/watch?v=b',
    ]


def test_get_missing_videos_views_order_no_view_counts(test_session):
    """When no entry has a `view_count` (old yt-dlp does not extract it from YouTube's lockupViewModel
    listings), the 'views' download order fails cleanly rather than downloading in arbitrary order."""
    from modules.videos.downloader import channel_downloader
    download = Download(
        url='https://www.youtube.com/@example/videos',
        settings={'download_order': 'views'},
        info_json={'entries': [
            {'_type': 'url', 'id': 'a', 'title': 'a', 'url': 'https://youtube.com/watch?v=a'},
            {'_type': 'url', 'id': 'b', 'title': 'b', 'url': 'https://youtube.com/watch?v=b'},
        ]},
    )
    test_session.add(download)
    test_session.commit()

    with pytest.raises(UnrecoverableDownloadError):
        channel_downloader.get_missing_videos(test_session, download)


@pytest.mark.asyncio
async def test_channel_download_crud(test_session, async_client, assert_downloads, tag_factory):
    """Test creating more complex Channel Download."""
    one, two = await tag_factory(), await tag_factory()
    download = {
        'urls': ['https://www.youtube.com/@wrolpi/videos'],
        'destination': '',
        'downloader': 'video_channel',
        'sub_downloader': 'video',
        'frequency': 2592000,
        'tag_names': [one.name, two.name],
        'settings': {
            'channel_tag_name': [one.name, ],
            'download_order': 'newest',
            'maximum_duration': 600,
            'minimum_duration': 60,
            'title_exclude': 'foo,bar',
            'title_include': 'baz,qux',
            'video_count_limit': 100,
            'video_resolutions': ['1080p', '720p', '480p', '360p', 'maximum'],
            'video_format': 'mp4',
        }
    }

    request, response = await async_client.post('/api/download', json=download)
    assert response.status == HTTPStatus.CREATED, request.body

    download = test_session.query(Download).one()
    assert download.url == 'https://www.youtube.com/@wrolpi/videos'
    assert download.destination is None
    assert download.tag_names == ['one', 'two']
    assert download.downloader == 'video_channel'
    assert download.sub_downloader == 'video'
    assert download.frequency == 2592000
    assert download.settings
    assert download.settings['channel_tag_name'] == ['one']
    assert download.settings['download_order'] == 'newest'
    assert download.settings['maximum_duration'] == 600
    assert download.settings['minimum_duration'] == 60
    assert download.settings['title_exclude'] == 'foo,bar'
    assert download.settings['title_include'] == 'baz,qux'
    assert download.settings['video_count_limit'] == 100
    assert download.settings['video_resolutions'] == ['1080p', '720p', '480p', '360p', 'maximum']
    assert download.settings['video_format'] == 'mp4'


def test_preview_filename(test_directory, fake_now):
    assert preview_filename('%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s') \
           == 'WROLPi_20000101_Qz-FuenRylQ_The title of the video.mp4'
    assert preview_filename('%(upload_date)s_%(id)s_%(title)s.%(ext)s') \
           == '20000101_Qz-FuenRylQ_The title of the video.mp4'
    assert preview_filename('%(upload_date)s_%(title)s.%(ext)s') \
           == '20000101_The title of the video.mp4'
    assert preview_filename('%(upload_date)s_%(description)s.%(ext)s') \
           == '20000101_A description of the video.mp4'
    assert preview_filename('%(timestamp)s_%(title)s.%(ext)s') \
           == '946684800_The title of the video.mp4'

    with pytest.raises(RuntimeError):
        preview_filename('%(upload_date)s_%(title)s.asdf')


def test_convert_wrolpi_filename_format():
    """WROLPi filename variables are converted to yt-dlp syntax."""
    from modules.videos.downloader import convert_wrolpi_filename_format

    # Test upload_year conversion
    assert convert_wrolpi_filename_format('%(upload_year)s/%(title)s.%(ext)s') \
           == '%(upload_date>%Y)s/%(title)s.%(ext)s'

    # Test upload_month conversion
    assert convert_wrolpi_filename_format('%(upload_month)s/%(title)s.%(ext)s') \
           == '%(upload_date>%m)s/%(title)s.%(ext)s'

    # Test both together
    assert convert_wrolpi_filename_format('%(upload_year)s/%(upload_month)s/%(title)s.%(ext)s') \
           == '%(upload_date>%Y)s/%(upload_date>%m)s/%(title)s.%(ext)s'

    # Test with other variables unchanged
    format_str = '%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
    expected = '%(upload_date>%Y)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
    assert convert_wrolpi_filename_format(format_str) == expected

    # Test format without WROLPi variables unchanged
    original = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
    assert convert_wrolpi_filename_format(original) == original


def test_preview_filename_with_upload_year(test_directory, fake_now):
    """preview_filename works with upload_year and upload_month variables."""
    # Year only
    assert preview_filename('%(upload_year)s/%(title)s.%(ext)s') \
           == '2000/The title of the video.mp4'

    # Month only
    assert preview_filename('%(upload_month)s/%(title)s.%(ext)s') \
           == '01/The title of the video.mp4'

    # Year and month together
    assert preview_filename('%(upload_year)s/%(upload_month)s/%(title)s.%(ext)s') \
           == '2000/01/The title of the video.mp4'

    # Combined with traditional format
    assert preview_filename('%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s') \
           == '2000/WROLPi_20000101_The title of the video.mp4'


def test_preview_filename_matches_youtubedl(test_directory, fake_now):
    """Ensure our lightweight preview_filename() matches YoutubeDL's actual output.

    This test validates that our simple string formatting produces identical
    results to YoutubeDL's prepare_filename(). If yt-dlp changes behavior,
    this test will catch it.
    """
    from copy import deepcopy
    from yt_dlp import YoutubeDL
    from wrolpi.dates import now

    test_formats = [
        '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
        '%(upload_date)s_%(id)s_%(title)s.%(ext)s',
        '%(upload_year)s/%(upload_month)s/%(title)s.%(ext)s',
        '%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s',
        '%(channel)s/%(id)s.%(ext)s',
    ]

    # Sample entry used by both methods - must match preview_filename's variables
    sample_date = now()
    entry = dict(
        uploader='WROLPi',
        channel='WROLPi',
        timestamp=int(sample_date.timestamp()),
        upload_date=sample_date.strftime('%Y%m%d'),
        id='Qz-FuenRylQ',
        title='The title of the video',
        ext='mp4',
        description='A description of the video',
    )

    for fmt in test_formats:
        # Get result from our lightweight implementation
        lightweight_result = preview_filename(fmt)

        # Get result from actual YoutubeDL
        converted_format = convert_wrolpi_filename_format(fmt)
        options = get_videos_downloader_config().yt_dlp_options
        options = deepcopy(options)
        options['outtmpl'] = converted_format
        ydl = YoutubeDL(options)
        ytdlp_result = prepare_filename(entry, ydl=ydl).lstrip('/')

        assert lightweight_result == ytdlp_result, \
            f"Mismatch for format '{fmt}':\n  lightweight: {lightweight_result}\n  ytdlp: {ytdlp_result}"


def test_normalize_video_file_names(test_directory, video_download_manager):
    """VideoDownloader renames caption files that do not share the video file's stem."""
    downloader = VideoDownloader()

    mp4 = test_directory / 'video.mp4'
    info_json = test_directory / 'video.info.json'

    # Minimal video files.
    vtt = test_directory / 'video.vtt'
    assert set(downloader.normalize_video_file_names(mp4, [mp4, vtt, info_json])) == {mp4, vtt, info_json}

    # Normal video files.
    vtt = test_directory / 'video.en.vtt'
    assert set(downloader.normalize_video_file_names(mp4, [mp4, vtt, info_json])) == {mp4, vtt, info_json}

    # Rare -auto file.
    vtt = test_directory / 'video.en-auto.vtt'
    assert set(downloader.normalize_video_file_names(mp4, [mp4, vtt, info_json])) == {mp4, vtt, info_json}

    # Extra characters get removed from the vtt file.
    vtt = test_directory / 'video.en-uYU-mmqFLq8.vtt'
    vtt.touch()  # must exist so it can be renamed.
    assert set(downloader.normalize_video_file_names(mp4, [mp4, vtt, info_json])) == {
        mp4,
        (test_directory / 'video.en.vtt'),
        info_json,
    }
    assert not vtt.exists()

    # VTT suffix that does not match expected pattern is left unchanged.
    vtt = test_directory / 'video.unknown_format.vtt'
    vtt.touch()
    assert set(downloader.normalize_video_file_names(mp4, [mp4, vtt, info_json])) == {mp4, vtt, info_json}
    assert vtt.exists()


@pytest.mark.asyncio
async def test_channel_downloader_sets_collection_id(test_session, channel_factory, mock_video_extract_info,
                                                     video_download_manager, await_switches):
    """Test that ChannelDownloader.do_download sets download.collection_id to the channel's collection.

    When a channel download runs, it should link the download to the channel's collection immediately.
    """
    channel = channel_factory(url='https://example.com/channel', source_id='UCtest123')

    download = Download(
        url='https://example.com/channel',
        downloader=channel_downloader.name,
        frequency=99,
    )
    test_session.add(download)
    test_session.commit()

    assert not download.collection_id, 'Download should not be linked yet'

    # Mock extract_info to return channel info
    mock_info = {
        'id': 'UCtest123',
        'channel_id': 'UCtest123',
        'uploader': channel.name,
        'entries': [],
        'webpage_url_basename': 'videos',
    }
    mock_video_extract_info.return_value = mock_info

    # Drive the phase-split API directly — no dispatch loop fires, so child video
    # downloads are never spawned and don't need stubbing.  refresh_files and
    # file_worker.wait_for_job are still patched because ChannelDownloader's
    # prepare_channel_for_downloads calls them inline.
    with mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
         mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
        from modules.archive.conftest import make_test_ctx
        prepared = channel_downloader.prepare_download(test_session, download)
        executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
        channel_downloader.finalize_download(test_session, download, executed)

    await await_switches()

    test_session.refresh(download)
    assert download.collection_id == channel.collection_id, \
        'ChannelDownloader should link download to channel collection'


@pytest.mark.asyncio
async def test_channel_passes_all_inheritable_settings_to_video(test_session, channel_factory,
                                                                 mock_video_extract_info,
                                                                 video_download_manager, await_switches):
    """ChannelDownloader passes all CHANNEL_INHERITABLE_SETTINGS to child video downloads."""
    from modules.videos.lib import CHANNEL_INHERITABLE_SETTINGS

    channel = channel_factory(url='https://example.com/channel', source_id='UCtest456')

    channel_settings = {
        'video_resolutions': ['720p'],
        'video_format': 'mkv',
        'writesubtitles': False,
        'sleep_requests': 3.0,
        'user_agent': 'TestBot/1.0',
        'video_codecs': ['h264'],
        'audio_codecs': ['aac'],
        'transcode': True,
        'strict_codecs': True,
    }

    download = Download(
        url='https://example.com/channel',
        downloader=channel_downloader.name,
        frequency=99,
        settings=channel_settings,
    )
    test_session.add(download)
    test_session.commit()

    mock_info = {
        'id': 'UCtest456',
        'channel_id': 'UCtest456',
        'uploader': channel.name,
        'entries': [{'id': 'vid1', 'url': 'https://youtube.com/watch?v=vid1', 'title': 'Video 1'}],
        'webpage_url_basename': 'videos',
    }
    mock_video_extract_info.return_value = mock_info

    # Direct phase-split call — child video downloads are never dispatched here.
    with mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
         mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
        from modules.archive.conftest import make_test_ctx
        prepared = channel_downloader.prepare_download(test_session, download)
        executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
        result = channel_downloader.finalize_download(test_session, download, executed)

    # Verify all channel-level inheritable settings are passed through
    for key in ('video_resolutions', 'video_format', 'writesubtitles', 'sleep_requests', 'user_agent',
                'video_codecs', 'audio_codecs', 'transcode', 'strict_codecs'):
        assert key in result.settings, f'{key} missing from DownloadResult.settings'
        assert result.settings[key] == channel_settings[key], \
            f'{key}: expected {channel_settings[key]}, got {result.settings[key]}'


@pytest.mark.asyncio
async def test_channel_does_not_pass_continue_dl_nooverwrites(test_session, channel_factory,
                                                               mock_video_extract_info,
                                                               video_download_manager, await_switches):
    """continue_dl and nooverwrites should NOT be passed from channel to video."""
    channel = channel_factory(url='https://example.com/channel', source_id='UCtest789')

    channel_settings = {
        'continue_dl': False,
        'nooverwrites': False,
        'video_format': 'mkv',
    }

    download = Download(
        url='https://example.com/channel',
        downloader=channel_downloader.name,
        frequency=99,
        settings=channel_settings,
    )
    test_session.add(download)
    test_session.commit()

    mock_info = {
        'id': 'UCtest789',
        'channel_id': 'UCtest789',
        'uploader': channel.name,
        'entries': [{'id': 'vid2', 'url': 'https://youtube.com/watch?v=vid2', 'title': 'Video 2'}],
        'webpage_url_basename': 'videos',
    }
    mock_video_extract_info.return_value = mock_info

    # Direct phase-split call — child video downloads are never dispatched here.
    with mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
         mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
        from modules.archive.conftest import make_test_ctx
        prepared = channel_downloader.prepare_download(test_session, download)
        executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
        result = channel_downloader.finalize_download(test_session, download, executed)

    assert 'continue_dl' not in result.settings, 'continue_dl should not be passed to child video downloads'
    assert 'nooverwrites' not in result.settings, 'nooverwrites should not be passed to child video downloads'
    assert result.settings['video_format'] == 'mkv', 'video_format should be passed through'


@pytest.mark.asyncio
async def test_video_download_uses_effective_settings(test_session, test_directory, mock_video_extract_info,
                                                      video_download_manager, mock_video_process_runner,
                                                      image_file, simple_channel, test_videos_downloader_config):
    """VideoDownloader uses merged effective settings from global + download overrides."""
    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)

    video_path = simple_channel.directory / 'a video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
    image_file.rename(video_path.with_suffix('.jpg'))

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'

    # Download with writesubtitles=False override (global default is True)
    settings = {'writesubtitles': False, 'sleep_requests': 5.0}

    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})

        video_download_manager.create_download(test_session, url, video_downloader.name, settings=settings)
        await video_download_manager.wait_for_all_downloads()

        mock_video_process_runner.assert_called_once()
        download, cmd, out_dir = mock_video_process_runner.call_args[0]

    # writesubtitles=False means no --write-subs in command
    assert '--write-subs' not in cmd, 'writesubtitles=False should prevent --write-subs'
    # sleep_requests=5.0 should appear
    assert '--sleep-requests' in cmd, 'sleep_requests override should appear in command'
    sleep_idx = cmd.index('--sleep-requests')
    assert cmd[sleep_idx + 1] == '5.0', f'Expected sleep_requests=5.0, got {cmd[sleep_idx + 1]}'


@pytest.mark.parametrize('video_resolutions', [
    ['1080p', '720p', '480p', 'maximum'],  # The default.
    ['480p'],
    ['2160p', 'maximum'],
])
def test_format_selector_contains_no_comma(video_resolutions):
    """`format_selector` joins selectors with "/" so yt-dlp downloads only the first match.

    A "," would tell yt-dlp to download *every* matching selector, which downloads and
    post-processes the same video once per match.
    """
    selector = format_selector(video_resolutions)

    assert ',' not in selector, f'A comma makes yt-dlp download every matching format: {selector}'
    # Every selector of every requested resolution is present, in order, as a "/" fallback.
    expected = [j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i]]
    assert selector == '/'.join(expected)


@pytest.mark.asyncio
async def test_video_download_requests_one_format(test_session, test_directory, mock_video_extract_info,
                                                  video_download_manager, mock_video_process_runner,
                                                  image_file, simple_channel, test_videos_downloader_config):
    """The `-f` passed to yt-dlp is a fallback chain, so only one format is downloaded."""
    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)

    video_path = simple_channel.directory / 'a video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
    image_file.rename(video_path.with_suffix('.jpg'))

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'

    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})

        video_download_manager.create_download(test_session, url, video_downloader.name)
        await video_download_manager.wait_for_all_downloads()

        mock_video_process_runner.assert_called_once()
        download, cmd, out_dir = mock_video_process_runner.call_args[0]

    assert '-f' in cmd, 'A video download must request formats'
    format_arg = cmd[cmd.index('-f') + 1]
    assert ',' not in format_arg, \
        f'A comma makes yt-dlp download the video once per matching format: {format_arg}'
    # The multiple resolutions the config requests are all still offered, as fallbacks.
    assert format_arg == format_selector(get_videos_downloader_config().video_resolutions)


@pytest.mark.asyncio
async def test_live_video_retry_seconds(test_session, test_directory, mock_video_extract_info,
                                        video_download_manager, mock_video_process_runner,
                                        simple_channel):
    """A live video download should return retry_seconds=3600 so it retries in 1 hour."""
    from wrolpi.cmd import CommandResult

    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)

    # Simulate yt-dlp output when a video is live.
    mock_video_process_runner.return_value = CommandResult(
        return_code=0, cancelled=False,
        stdout=b'does not pass filter (!is_live), skipping ..', stderr=b'', elapsed=0,
    )

    url = 'https://www.youtube.com/watch?v=live_video'
    mock_video_extract_info.return_value = example_video_json

    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        # Return a path that won't exist (video wasn't downloaded because it's live).
        mock_prepare_filename.return_value = (simple_channel.directory / 'live video.mp4', {'id': 'foo'})

        download = Download(url=url, downloader='video', attempts=0)
        # Drive the phase-split API directly: when execute_download detects a live video
        # it short-circuits with a DownloadResult (the dispatch surfaces it verbatim and
        # skips finalize_download).
        from modules.archive.conftest import make_test_ctx
        prepared = video_downloader.prepare_download(test_session, download)
        result = await video_downloader.execute_download(prepared, make_test_ctx(), download=download)

    assert result.success is False
    assert result.retry_seconds == 3600
    assert 'live' in result.error.lower()


def test_bot_blocked_detection():
    """Bot detection errors should be identified by _bot_blocked."""
    assert _bot_blocked("Sign in to confirm you're not a bot") is True
    assert _bot_blocked("Sign in to confirm you\u2019re not a bot") is True
    assert _bot_blocked("cookies are no longer valid") is True
    assert _bot_blocked("ERROR: Sign in to confirm you're not a bot. Use --cookies") is True

    # These should NOT be detected as bot-blocked.
    assert _bot_blocked("404: Not Found") is False
    assert _bot_blocked("requires payment") is False
    assert _bot_blocked("some random error") is False


def test_skip_download_excludes_bot_blocked():
    """Bot-blocked errors should NOT cause a video to be skipped (added to skip list)."""
    # Bot-blocked errors should not be skipped (they should stop all downloads instead).
    assert _skip_download("Sign in to confirm you're not a bot") is False
    assert _skip_download("cookies are no longer valid") is False

    # Regular "Sign in" errors that are NOT bot-related should still be skipped.
    assert _skip_download("Sign in to view this video") is True

    # Other unrecoverable errors should still be skipped.
    assert _skip_download("404: Not Found") is True
    assert _skip_download("requires payment") is True


@pytest.mark.asyncio
async def test_video_download_playlist_appended_extension(test_session, test_directory, mock_video_extract_info,
                                                          video_download_manager, mock_video_process_runner,
                                                          simple_channel, image_file):
    """A video download from a playlist (e.g. Twitter/X) where yt-dlp appends the real extension to a bogus .NA
    extension should still find the file."""
    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)

    # Simulate yt-dlp saving a file with .NA.mp4 (appended extension from playlist entry with ext=NA).
    video_path_na = simple_channel.directory / 'uploader name_20190707_some long id_The video title.NA.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path_na)

    url = 'https://x.com/example/status/123'
    mock_video_extract_info.return_value = example_video_json

    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        # prepare_filename returns the path without the real extension (ending in .NA).
        bogus_path = video_path_na.parent / video_path_na.stem  # Removes .mp4, leaving .NA as "extension"
        mock_prepare_filename.return_value = (bogus_path, {'id': 'foo'})

        video_download_manager.create_download(test_session, url, video_downloader.name)
        await video_download_manager.wait_for_all_downloads()

    video: Video = test_session.query(Video).one()
    assert video.video_path == video_path_na


def test_parse_ytdlp_progress():
    """yt-dlp progress lines are parsed correctly."""
    line = '[download]  50.0% of ~  45.67MiB at  2.34MiB/s ETA 00:19'
    result = parse_ytdlp_progress(line)
    assert result is not None
    total = int(45.67 * 1024 ** 2)
    assert result['total_bytes'] == total
    assert result['percent'] == 50
    assert result['bytes_downloaded'] == int(0.5 * total)
    assert result['speed'] == int(2.34 * 1024 ** 2)
    assert result['eta'] == '00:19'


def test_parse_ytdlp_progress_no_estimated():
    """yt-dlp progress lines without ~ estimated marker are parsed."""
    line = '[download] 100% of   45.67MiB at  3.45MiB/s ETA 00:00'
    result = parse_ytdlp_progress(line)
    assert result is not None
    assert result['percent'] == 100
    assert result['total_bytes'] == int(45.67 * 1024 ** 2)
    assert result['speed'] == int(3.45 * 1024 ** 2)
    assert result['eta'] == '00:00'


def test_parse_ytdlp_progress_small_percent():
    """yt-dlp progress lines with small percentages are parsed."""
    line = '[download]   0.1% of ~  45.67MiB at  1.23MiB/s ETA 00:37'
    result = parse_ytdlp_progress(line)
    assert result is not None
    assert result['percent'] == 0
    assert result['eta'] == '00:37'


def test_parse_ytdlp_progress_completion_line():
    """The final yt-dlp line with 'in' instead of 'at/ETA' returns None (no speed/ETA)."""
    line = '[download] 100% of   45.67MiB in 00:13'
    result = parse_ytdlp_progress(line)
    assert result is None


def test_parse_ytdlp_progress_not_progress_line():
    """Non-progress lines return None."""
    assert parse_ytdlp_progress('[info] Downloading video 1 of 1') is None
    assert parse_ytdlp_progress('') is None
    assert parse_ytdlp_progress('[download] Destination: video.mp4') is None


@pytest.mark.asyncio
async def test_fetch_video_duration_persistent_cache(test_download_cache_config):
    """Video durations are persisted to the download cache config and reused on subsequent calls."""
    from modules.videos.downloader import _cache_video_duration
    from wrolpi.downloader import get_download_cache_config

    url = 'https://www.youtube.com/watch?v=test123'

    # Verify cache is initially empty.
    assert get_download_cache_config().video_durations == []

    # Persist a duration directly.
    _cache_video_duration(url, 120)
    assert get_download_cache_config().video_durations == [[url, 120]]

    # Re-caching the same URL updates the entry (no duplicate).
    _cache_video_duration(url, 200)
    assert get_download_cache_config().video_durations == [[url, 200]]


@pytest.mark.asyncio
async def test_fetch_video_duration_cache_eviction(test_download_cache_config):
    """Video duration cache evicts oldest entries when over the max size."""
    from modules.videos.downloader import _cache_video_duration, VIDEO_DURATION_CACHE_MAX_SIZE
    from wrolpi.downloader import get_download_cache_config

    # Fill to max size.
    for i in range(VIDEO_DURATION_CACHE_MAX_SIZE):
        _cache_video_duration(f'https://example.com/video{i}', i)
    assert len(get_download_cache_config().video_durations) == VIDEO_DURATION_CACHE_MAX_SIZE

    # Adding one more should evict the oldest (video0), keeping it at max size.
    _cache_video_duration('https://example.com/new_video', 999)
    durations = get_download_cache_config().video_durations
    assert len(durations) == VIDEO_DURATION_CACHE_MAX_SIZE
    # Newest entry is at the end.
    assert durations[-1] == ['https://example.com/new_video', 999]
    # Oldest entry (video0) was evicted.
    durations_dict = dict(durations)
    assert 'https://example.com/video0' not in durations_dict
    # Second oldest (video1) is still present.
    assert durations_dict['https://example.com/video1'] == 1


@pytest.mark.asyncio
async def test_channel_download_preserves_skip_list(test_session, test_directory, simple_channel,
                                                    video_download_manager, video_file,
                                                    mock_video_extract_info, mock_video_prepare_filename,
                                                    await_switches, mock_video_process_runner,
                                                    test_download_manager_config):
    """A skipped video URL should remain in the skip list when its channel downloads again.

    Regression: child video downloads created by ChannelDownloader used reset_attempts=True which
    caused create_downloads to remove the URL from the skip list."""
    skipped_url = 'https://youtube.com/watch?v=video_1_url'
    url = 'https://www.youtube.com/c/ExampleChannel/videos'

    # Skip video_1_url — simulating user skipping a video.
    get_download_manager_config().skip_urls = [skipped_url]
    assert get_download_manager_config().skip_urls == [skipped_url]

    mock_video_prepare_filename.return_value = video_file
    mock_video_extract_info.return_value = example_channel_json
    with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
        mock_get_channel.return_value = simple_channel
        simple_channel.get_or_create_download(test_session, url, 60)
        # Download channel catalog.
        await video_download_manager.wait_for_all_downloads()
        # Download the child videos.
        await video_download_manager.wait_for_all_downloads()

    # The skipped URL must still be in the skip list.
    assert skipped_url in get_download_manager_config().skip_urls, \
        'Channel download removed a URL from the skip list!'

    # Only video_2 should have been downloaded, not the skipped video_1.
    once_downloads = video_download_manager.get_once_downloads(test_session)
    video_downloads = [i for i in once_downloads if 'watch' in i.url]
    assert {i.url for i in video_downloads} == {'https://youtube.com/watch?v=video_2_url'}


# ---------------------------------------------------------------------------
# Phase-split unit tests for VideoDownloader.prepare_download.
#
# These exercise the sync prep invariants in isolation — no async_client, no
# extract_info, no subprocess.  Heavier execute/finalize behavior is covered by
# the integration tests above (which drive through video_download_manager).
# ---------------------------------------------------------------------------
from modules.videos.downloader import PreparedVideo  # noqa: E402


def test_prepare_max_attempts_raises(test_session):
    """prepare_download enforces the 10-attempt cap (videos retry more than other
    downloaders because YouTube transients are common)."""
    download = Download(url='https://youtube.com/watch?v=foo', downloader='video', attempts=10)

    with pytest.raises(UnrecoverableDownloadError, match='Max download attempts'):
        video_downloader.prepare_download(test_session, download)


def test_prepare_normalizes_url(test_session):
    """Shorts URLs become watch URLs in the PreparedVideo (matches DB-stored format)."""
    download = Download(
        url='https://www.youtube.com/shorts/ABC123',
        downloader='video',
        settings={},
    )

    prepared = video_downloader.prepare_download(test_session, download)

    assert isinstance(prepared, PreparedVideo)
    assert prepared.url == 'https://www.youtube.com/watch?v=ABC123'


def test_prepare_copies_upstream_settings_onto_download(test_session):
    """ChannelDownloader passes destination + tag_names via settings; prep copies them
    onto the Download row so downstream code sees them as columns."""
    download = Download(
        url='https://youtube.com/watch?v=foo',
        downloader='video',
        destination=None,
        tag_names=None,
        settings={'destination': '/media/wrolpi/videos/foo', 'tag_names': ['news', 'tech']},
    )

    prepared = video_downloader.prepare_download(test_session, download)

    assert download.destination == '/media/wrolpi/videos/foo'
    assert download.tag_names == ['news', 'tech']
    assert prepared.tag_names == ['news', 'tech']


def test_prepare_uses_existing_video_location(test_session, video_factory):
    """If a Video already exists at the URL, prep captures its location for error
    attribution (so a failed re-download still tells the user where the previous file is)."""
    url = 'https://youtube.com/watch?v=existing'
    existing = video_factory(with_video_file=True)
    existing.file_group.url = url
    test_session.commit()

    download = Download(url=url, downloader='video', settings={})

    prepared = video_downloader.prepare_download(test_session, download)

    assert prepared.location == existing.location
    assert prepared.location is not None


@pytest.mark.parametrize('url,expected', [
    ('https://www.youtube.com/watch?v=abc123', True),
    ('https://youtube.com/watch?v=abc123', True),
    ('https://m.youtube.com/watch?v=abc123', True),
    ('https://music.youtube.com/watch?v=abc123', True),
    ('https://youtu.be/abc123', True),
    # Lookalike / malicious hosts must be rejected.
    ('https://youtube.com.example.org/watch?v=abc123', False),
    ('https://notyoutube.com/watch?v=abc123', False),
    ('https://vimeo.com/12345', False),
    ('https://example.com/youtube.com', False),
    ('', False),
    (None, False),
])
def test_is_youtube_url(url, expected):
    """is_youtube_url matches YouTube hosts (and subdomains) but rejects lookalike hosts."""
    assert is_youtube_url(url) is expected


def test_format_selector_with_codecs():
    """Codec preferences generate codec-major selectors: a preferred codec at a lower resolution
    beats a non-preferred codec at a higher resolution.  The legacy resolution selectors are
    appended as a fallback unless strict."""
    from modules.videos.downloader import VIDEO_CODEC_YT_DLP_PATTERNS, AUDIO_CODEC_YT_DLP_PATTERNS

    h264 = VIDEO_CODEC_YT_DLP_PATTERNS['h264']
    aac = AUDIO_CODEC_YT_DLP_PATTERNS['aac']
    legacy = format_selector(['1080p', '720p'])

    selector = format_selector(['1080p', '720p'], video_codecs=['h264'], audio_codecs=['aac'])
    expected = '/'.join([
        f"bestvideo[height=1080][vcodec~='{h264}']+bestaudio[acodec~='{aac}']",
        f"bestvideo[height=1080][vcodec~='{h264}']+bestaudio",
        f"bestvideo[width=1080][vcodec~='{h264}']+bestaudio[acodec~='{aac}']",
        f"bestvideo[width=1080][vcodec~='{h264}']+bestaudio",
        f"bestvideo[height=720][vcodec~='{h264}']+bestaudio[acodec~='{aac}']",
        f"bestvideo[height=720][vcodec~='{h264}']+bestaudio",
        f"bestvideo[width=720][vcodec~='{h264}']+bestaudio[acodec~='{aac}']",
        f"bestvideo[width=720][vcodec~='{h264}']+bestaudio",
        legacy,
    ])
    assert selector == expected
    assert ',' not in selector

    # Strict drops the any-codec audio fallbacks and the legacy tail.
    strict = format_selector(['1080p', '720p'], video_codecs=['h264'], audio_codecs=['aac'],
                             strict_codecs=True)
    expected_strict = '/'.join([
        f"bestvideo[height=1080][vcodec~='{h264}']+bestaudio[acodec~='{aac}']",
        f"bestvideo[width=1080][vcodec~='{h264}']+bestaudio[acodec~='{aac}']",
        f"bestvideo[height=720][vcodec~='{h264}']+bestaudio[acodec~='{aac}']",
        f"bestvideo[width=720][vcodec~='{h264}']+bestaudio[acodec~='{aac}']",
    ])
    assert strict == expected_strict
    assert ',' not in strict

    # Codec-major: every h264 selector comes before the first vp9 selector.
    vp9 = VIDEO_CODEC_YT_DLP_PATTERNS['vp9']
    two = format_selector(['1080p'], video_codecs=['h264', 'vp9'], strict_codecs=True)
    assert two.rindex(h264) < two.index(vp9)
    assert two.count('bestvideo') == 4  # (height + width) x 2 codecs, one audio selector each.

    # No codecs at all: byte-identical to the legacy selector.
    assert format_selector(['1080p', '720p']) == legacy

    # Only audio codecs: video is unfiltered but audio is preferred.
    audio_only_pref = format_selector(['720p'], audio_codecs=['opus'])
    assert "bestvideo[height=720]+bestaudio[acodec~='^opus']" in audio_only_pref
    assert audio_only_pref.endswith(format_selector(['720p']))

    # 'maximum' has no dimension filter.
    maximum = format_selector(['maximum'], video_codecs=['h264'], strict_codecs=True)
    assert maximum == f"bestvideo[vcodec~='{h264}']+bestaudio"


def test_format_selector_codec_patterns_are_safe():
    """Codec regex patterns are embedded in `-f` bracket filters; a comma would make yt-dlp
    download every matching format, and a slash would break the fallback chain."""
    from modules.videos.downloader import VIDEO_CODEC_YT_DLP_PATTERNS, AUDIO_CODEC_YT_DLP_PATTERNS

    for pattern in (*VIDEO_CODEC_YT_DLP_PATTERNS.values(), *AUDIO_CODEC_YT_DLP_PATTERNS.values()):
        assert ',' not in pattern
        assert '/' not in pattern


@pytest.mark.asyncio
async def test_video_download_requests_codec_format(test_session, test_directory, mock_video_extract_info,
                                                    video_download_manager, mock_video_process_runner,
                                                    image_file, simple_channel, test_videos_downloader_config):
    """Codec settings change the `-f` selector; the legacy selectors remain as fallbacks."""
    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)

    video_path = simple_channel.directory / 'a video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
    image_file.rename(video_path.with_suffix('.jpg'))

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
    settings = {'video_codecs': ['h264'], 'audio_codecs': ['aac']}

    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})

        video_download_manager.create_download(test_session, url, video_downloader.name, settings=settings)
        await video_download_manager.wait_for_all_downloads()

        mock_video_process_runner.assert_called_once()
        download, cmd, out_dir = mock_video_process_runner.call_args[0]

    format_arg = cmd[cmd.index('-f') + 1]
    resolutions = get_videos_downloader_config().video_resolutions
    assert format_arg == format_selector(resolutions, video_codecs=['h264'], audio_codecs=['aac'])
    assert "vcodec~=" in format_arg
    # The legacy selectors are still offered as a fallback (strict is off).
    assert format_arg.endswith(format_selector(resolutions))


@pytest.mark.asyncio
async def test_strict_selector_when_transcode_cannot_satisfy(test_session, test_directory,
                                                             mock_video_extract_info,
                                                             video_download_manager,
                                                             mock_video_process_runner, image_file,
                                                             simple_channel,
                                                             test_videos_downloader_config):
    """Transcode suppresses strict only when it can produce a preferred codec; av1 has no
    transcode target, so strict still removes the fallback selectors."""
    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)

    video_path = simple_channel.directory / 'a video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
    image_file.rename(video_path.with_suffix('.jpg'))

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
    settings = {'video_codecs': ['av1'], 'strict_codecs': True, 'transcode': True}

    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename, \
            mock.patch('modules.videos.downloader.enforce_codecs', new_callable=mock.AsyncMock) as mock_enforce:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
        mock_enforce.side_effect = lambda path, paths, effective: (path, paths)

        video_download_manager.create_download(test_session, url, video_downloader.name, settings=settings)
        await video_download_manager.wait_for_all_downloads()

        mock_video_process_runner.assert_called_once()
        download, cmd, out_dir = mock_video_process_runner.call_args[0]

    format_arg = cmd[cmd.index('-f') + 1]
    resolutions = get_videos_downloader_config().video_resolutions
    assert format_arg == format_selector(resolutions, video_codecs=['av1'], strict_codecs=True)
    # No fallback to the legacy selectors.
    assert not format_arg.endswith(format_selector(resolutions))


@pytest.mark.asyncio
async def test_strict_codecs_unavailable_fails_download(test_session, test_directory, mock_video_extract_info,
                                                        video_download_manager, mock_video_process_runner,
                                                        image_file, simple_channel,
                                                        test_videos_downloader_config):
    """With strict_codecs, yt-dlp's "Requested format is not available" fails the download
    permanently (the source will never grow new formats)."""
    from wrolpi.cmd import CommandResult

    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)
    video_path = simple_channel.directory / 'a video.mp4'

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
    settings = {'video_codecs': ['h264'], 'strict_codecs': True}

    mock_video_process_runner.return_value = CommandResult(
        return_code=1, cancelled=False, stdout=b'',
        stderr=b'ERROR: [youtube] 31jPEBiAC3c: Requested format is not available.', elapsed=0)

    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})

        video_download_manager.create_download(test_session, url, video_downloader.name, settings=settings)
        await video_download_manager.wait_for_all_downloads()

    download = test_session.query(Download).one()
    assert download.is_failed, f'Download should have failed permanently, got {download.status}'
    assert 'transcoding cannot produce them' in download.error


@pytest.mark.asyncio
async def test_codecs_unavailable_defers_without_strict(test_session, test_directory, mock_video_extract_info,
                                                        video_download_manager, mock_video_process_runner,
                                                        image_file, simple_channel,
                                                        test_videos_downloader_config):
    """The same yt-dlp failure without strict_codecs is deferred (retried), as before."""
    from wrolpi.cmd import CommandResult

    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)
    video_path = simple_channel.directory / 'a video.mp4'

    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'

    mock_video_process_runner.return_value = CommandResult(
        return_code=1, cancelled=False, stdout=b'',
        stderr=b'ERROR: [youtube] 31jPEBiAC3c: Requested format is not available.', elapsed=0)

    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})

        video_download_manager.create_download(test_session, url, video_downloader.name,
                                               settings={'video_codecs': ['h264']})
        await video_download_manager.wait_for_all_downloads()

    download = test_session.query(Download).one()
    assert download.is_deferred, f'Download should defer without strict_codecs, got {download.status}'


@pytest.mark.asyncio
async def test_enforce_codecs():
    """enforce_codecs transcodes, fails, or keeps the file depending on the effective settings."""
    import pathlib
    from modules.videos.downloader import enforce_codecs

    video_path = pathlib.Path('/tmp/video.webm')
    poster_path = pathlib.Path('/tmp/video.jpg')
    video_paths = [video_path, poster_path]
    vp9_ffprobe = {'streams': [
        {'codec_type': 'video', 'codec_name': 'vp9'},
        {'codec_type': 'audio', 'codec_name': 'opus'},
    ]}

    with mock.patch('modules.videos.downloader.get_or_create_ffprobe_json',
                    new_callable=mock.AsyncMock) as mock_ffprobe, \
            mock.patch('modules.videos.downloader.transcode_video_file',
                       new_callable=mock.AsyncMock) as mock_transcode:
        mock_ffprobe.return_value = (vp9_ffprobe, None)

        # Codecs match: nothing happens.
        effective = {'video_codecs': ['vp9'], 'audio_codecs': ['opus']}
        result = await enforce_codecs(video_path, video_paths, effective)
        assert result == (video_path, video_paths)
        mock_transcode.assert_not_called()

        # Mismatch with neither transcode nor strict: file is kept.
        effective = {'video_codecs': ['h264']}
        result = await enforce_codecs(video_path, video_paths, effective)
        assert result == (video_path, video_paths)
        mock_transcode.assert_not_called()

        # Mismatch with strict: permanent failure.
        effective = {'video_codecs': ['h264'], 'strict_codecs': True}
        with pytest.raises(UnrecoverableDownloadError):
            await enforce_codecs(video_path, video_paths, effective)
        mock_transcode.assert_not_called()

        # Mismatch with transcode: the file is transcoded and the paths updated.
        new_path = video_path.with_suffix('.mp4')
        mock_transcode.return_value = new_path
        effective = {'video_codecs': ['h264'], 'audio_codecs': ['aac'], 'transcode': True,
                     'video_format': 'mp4'}
        result = await enforce_codecs(video_path, video_paths, effective)
        assert result == (new_path, [new_path, poster_path])
        mock_transcode.assert_called_once_with(video_path, 'h264', 'aac', container='mp4')

        # Transcode wins over strict when it can produce the preferred codec.
        mock_transcode.reset_mock()
        mock_transcode.return_value = new_path
        effective = {'video_codecs': ['h264'], 'transcode': True, 'strict_codecs': True,
                     'video_format': 'mp4'}
        result = await enforce_codecs(video_path, video_paths, effective)
        assert result[0] == new_path
        mock_transcode.assert_called_once()

        # No supported transcode target: file is kept, no error.
        mock_transcode.reset_mock()
        effective = {'video_codecs': ['av1'], 'transcode': True}
        result = await enforce_codecs(video_path, video_paths, effective)
        assert result == (video_path, video_paths)
        mock_transcode.assert_not_called()

        # Regression: strict must hold when transcode cannot produce the preferred codec.
        effective = {'video_codecs': ['av1'], 'transcode': True, 'strict_codecs': True}
        with pytest.raises(UnrecoverableDownloadError):
            await enforce_codecs(video_path, video_paths, effective)
        mock_transcode.assert_not_called()

        # No partial transcode: an unfixable video mismatch (av1) blocks the audio-only
        # conversion; the file is kept as-is (or fails when strict).
        effective = {'video_codecs': ['av1'], 'audio_codecs': ['aac'], 'transcode': True}
        result = await enforce_codecs(video_path, video_paths, effective)
        assert result == (video_path, video_paths)
        mock_transcode.assert_not_called()
        effective = {'video_codecs': ['av1'], 'audio_codecs': ['aac'], 'transcode': True,
                     'strict_codecs': True}
        with pytest.raises(UnrecoverableDownloadError):
            await enforce_codecs(video_path, video_paths, effective)
        mock_transcode.assert_not_called()


@pytest.mark.asyncio
async def test_metadata_refresh_keeps_wrolpi_section(test_session, test_directory, video_factory):
    """Refreshing a Video's metadata replaces yt-dlp's info json; the wrolpi section (the user's
    custom title and description) is carried over and still wins."""
    source_id = 'video-source-id'
    url = 'https://example.com/the-video'
    video = video_factory(with_info_json={'id': source_id, 'title': 'Orig',
                                          'wrolpi': {'custom_title': 'Mine', 'custom_description': 'my words'}},
                          source_id=source_id)
    video.file_group.url = url
    test_session.commit()
    assert video.file_group.title == 'Mine'

    download = Download(url=url, info_json={'id': source_id}, settings={})
    with mock.patch('modules.videos.downloader.download_video_info_json') as mock_download:
        mock_download.return_value = {'id': source_id, 'title': 'Orig v2', 'description': 'theirs', 'duration': 9}
        result = await VideoDownloader.download_info_json(download, video.video_path)
    assert result.success is True, result

    test_session.expire_all()
    video = Video.find_by_file_group_id(test_session, video.file_group_id)
    info_json = video.get_info_json()
    assert info_json['title'] == 'Orig v2', 'yt-dlp data was refreshed'
    assert info_json['wrolpi'] == {'custom_title': 'Mine', 'custom_description': 'my words'}
    assert video.file_group.title == 'Mine'
    assert video.get_description() == 'my words'


@pytest.mark.asyncio
async def test_redownload_keeps_wrolpi_section(test_session, test_directory, mock_video_extract_info, simple_channel,
                                               await_switches, video_download_manager, mock_video_process_runner,
                                               image_file):
    """A full re-download has yt-dlp overwrite the info json; the wrolpi section is carried over."""
    simple_channel.source_id = example_video_json['channel_id']
    simple_channel.directory = test_directory / 'videos/channel name'
    simple_channel.directory.mkdir(parents=True)
    url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'

    video_path = simple_channel.directory / 'a video.mp4'
    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
    info_json_path = video_path.with_suffix('.info.json')
    info_json_path.write_text(json.dumps({'title': 'Orig', 'webpage_url': url, 'wrolpi': {'custom_title': 'Mine'}}))
    from modules.videos.normalize_video_url import normalize_video_url
    existing = Video.from_paths(test_session, video_path, info_json_path)
    existing.validate(test_session)
    test_session.commit()
    assert existing.file_group.title == 'Mine'
    assert Video.get_by_url(test_session, normalize_video_url(url)) is not None, 'lookup by normalized url'
    assert existing.file_group.get_wrolpi_json() == {'custom_title': 'Mine'}

    from wrolpi.cmd import CommandResult

    async def yt_dlp_overwrites_info_json(*args, **kwargs):
        # yt-dlp rewrites the whole info json; the wrolpi section is gone from disk.
        info_json_path.write_text(json.dumps({'title': 'Orig v2', 'webpage_url': url, 'duration': 5}))
        return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)

    mock_video_process_runner.side_effect = yt_dlp_overwrites_info_json
    with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
        mock_video_extract_info.return_value = example_video_json
        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
        video_download_manager.create_download(test_session, url, video_downloader.name)
        await video_download_manager.wait_for_all_downloads()
        mock_video_process_runner.assert_called_once()

    test_session.expire_all()
    video: Video = test_session.query(Video).one()
    info_json = video.get_info_json()
    assert info_json['title'] == 'Orig v2'
    assert info_json['wrolpi'] == {'custom_title': 'Mine'}
    assert video.file_group.title == 'Mine'