mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Video download destination can download to the incorrect Channel.
This commit is contained in:
parent
bbc23dbf90
commit
105ae377f9
2 changed files with 107 additions and 66 deletions
|
|
@ -611,15 +611,17 @@ class VideoDownloader(Downloader, ABC):
|
|||
if not download.info_json:
|
||||
raise ValueError(f'Cannot download video with no info_json.')
|
||||
|
||||
channel, channel_directory, channel_id, destination, settings = await self._get_channel(session, download)
|
||||
channel = await self._get_channel(session, download)
|
||||
channel_id = channel.id if channel else None
|
||||
destination = download.destination
|
||||
|
||||
if destination:
|
||||
# Download to the directory specified in the settings.
|
||||
out_dir = destination
|
||||
logger.debug(f'Downloading {url} to destination {destination} from settings')
|
||||
elif channel:
|
||||
out_dir = channel_directory
|
||||
logger.debug(f'Downloading {url} to channel directory: {channel_directory}')
|
||||
out_dir = channel.directory
|
||||
logger.debug(f'Downloading {url} to channel directory: {channel.directory}')
|
||||
else:
|
||||
# Download to the default directory if this video has no channel.
|
||||
out_dir = get_no_channel_directory()
|
||||
|
|
@ -633,7 +635,6 @@ class VideoDownloader(Downloader, ABC):
|
|||
if tag_names and logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(f'Downloading {url} with {tag_names=}')
|
||||
|
||||
config = get_videos_downloader_config()
|
||||
effective = get_effective_video_settings(settings)
|
||||
|
||||
logs = None # noqa
|
||||
|
|
@ -859,71 +860,59 @@ class VideoDownloader(Downloader, ABC):
|
|||
return result
|
||||
|
||||
@staticmethod
|
||||
async def _get_channel(session: Session, download: Download) \
|
||||
-> Tuple[Optional[Channel], Optional[pathlib.Path], Optional[int], Optional[pathlib.Path], dict]:
|
||||
settings = download.settings or dict()
|
||||
channel_tag_name = i[0] if (i := settings.get('channel_tag_name')) else None
|
||||
async def _get_channel(session: Session, download: Download) -> Optional[Channel]:
|
||||
"""Find the Channel for a video download.
|
||||
|
||||
found_channel = None
|
||||
# Look for Channel using channel data first, fallback to uploader data.
|
||||
Priority order:
|
||||
1. Destination directory — if it belongs to a channel, that channel wins (files must live in their channel's dir)
|
||||
2. yt-dlp info_json metadata (channel/uploader fields)
|
||||
3. Download settings (collection_id or channel_url from ChannelDownloader)
|
||||
"""
|
||||
settings = download.settings or dict()
|
||||
destination = download.destination
|
||||
|
||||
# Destination channel always takes priority — files must live in their channel's directory.
|
||||
if destination:
|
||||
try:
|
||||
channel = get_channel(session, directory=destination, return_dict=False)
|
||||
logger.debug(f'Found {channel} using destination directory')
|
||||
return channel
|
||||
except UnknownChannel:
|
||||
pass
|
||||
|
||||
# Try yt-dlp info_json metadata.
|
||||
channel_name = download.info_json.get('channel') or download.info_json.get('uploader')
|
||||
channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
|
||||
channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
|
||||
channel = None
|
||||
if channel_name or channel_source_id or channel_url:
|
||||
# Try to find the channel via info_json from yt-dlp.
|
||||
tag_name = i[0] if (i := settings.get('channel_tag_name')) else None
|
||||
try:
|
||||
channel = get_or_create_channel(session, source_id=channel_source_id, url=channel_url,
|
||||
name=channel_name, tag_name=channel_tag_name,
|
||||
destination=download.destination)
|
||||
if channel:
|
||||
found_channel = 'yt_dlp'
|
||||
logger.debug(f'Found a channel with source_id {channel=}')
|
||||
name=channel_name, tag_name=tag_name, destination=destination)
|
||||
logger.debug(f'Found {channel} using yt-dlp info_json')
|
||||
return channel
|
||||
except UnknownChannel:
|
||||
# Can't find a channel, use the no channel directory.
|
||||
pass
|
||||
destination = download.destination
|
||||
if not channel and destination:
|
||||
# Destination may find Channel if not already found.
|
||||
try:
|
||||
channel = get_channel(session, directory=destination, return_dict=False)
|
||||
found_channel = 'download_settings_directory'
|
||||
logger.debug(f'Found a channel that shares the destination directory {channel=}')
|
||||
except UnknownChannel:
|
||||
# Destination must not be a channel.
|
||||
pass
|
||||
# Look up Channel via Collection or channel_url from settings
|
||||
|
||||
# Try collection_id or channel_url from ChannelDownloader settings.
|
||||
collection_id = download.collection_id
|
||||
channel_url = settings.get('channel_url')
|
||||
if not channel and (collection_id or channel_url):
|
||||
# Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
|
||||
logger.info(f'Using download.settings to find channel')
|
||||
settings_channel_url = settings.get('channel_url')
|
||||
if collection_id:
|
||||
channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
|
||||
if channel:
|
||||
logger.debug(f'Found {channel} using collection_id')
|
||||
return channel
|
||||
if settings_channel_url:
|
||||
try:
|
||||
# Find Channel by collection_id or URL
|
||||
if collection_id:
|
||||
channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
|
||||
if not channel and channel_url:
|
||||
channel = get_channel(session, url=channel_url, return_dict=False)
|
||||
logger.debug(f'Found channel with collection_id or channel url {channel=}')
|
||||
channel = get_channel(session, url=settings_channel_url, return_dict=False)
|
||||
logger.debug(f'Found {channel} using settings channel_url')
|
||||
return channel
|
||||
except UnknownChannel:
|
||||
# We do not need a Channel if we have a destination directory.
|
||||
if not destination:
|
||||
raise
|
||||
if channel:
|
||||
found_channel = 'download_settings'
|
||||
|
||||
channel_id = channel.id if channel else None
|
||||
channel_directory = channel.directory if channel else None
|
||||
if found_channel == 'yt_dlp':
|
||||
logger.debug(f'Found {channel} using yt_dlp')
|
||||
elif found_channel == 'download_settings':
|
||||
logger.info(f'Found {channel} using Download.settings')
|
||||
elif found_channel == 'download_settings_directory':
|
||||
logger.info(f'Found {channel} using Download.settings directory')
|
||||
else:
|
||||
logger.warning('Could not find channel')
|
||||
|
||||
return channel, channel_directory, channel_id, destination, settings
|
||||
logger.warning('Could not find channel')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def prepare_filename(url: str, out_dir: pathlib.Path, video_resolutions: str, video_format: str) \
|
||||
|
|
@ -1113,6 +1102,17 @@ def get_or_create_channel(session: Session, source_id: str = None, url: str = No
|
|||
except UnknownChannel:
|
||||
pass
|
||||
|
||||
# If destination matches an existing channel, use it rather than creating a new one.
|
||||
# This handles the case where yt-dlp returns bad metadata (e.g. channel name 'watch')
|
||||
# but the destination directory already belongs to a known channel.
|
||||
if destination:
|
||||
try:
|
||||
channel = get_channel(session, directory=destination, return_dict=False)
|
||||
channel.dict()
|
||||
return channel
|
||||
except UnknownChannel:
|
||||
pass
|
||||
|
||||
if not name:
|
||||
raise UnknownChannel(f'Cannot create channel without a name')
|
||||
|
||||
|
|
|
|||
|
|
@ -322,8 +322,8 @@ async def test_get_or_create_channel_with_destination(async_client, test_session
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_or_create_channel_destination_conflict(async_client, test_session, test_directory):
|
||||
"""Creating a channel at a destination that conflicts with an existing channel should raise
|
||||
UnrecoverableDownloadError."""
|
||||
"""When destination matches an existing channel's directory, that channel should be returned
|
||||
instead of trying to create a new one (which would raise ChannelDirectoryConflict)."""
|
||||
from wrolpi.collections import Collection
|
||||
|
||||
# Create an existing channel at the destination directory.
|
||||
|
|
@ -336,16 +336,15 @@ async def test_get_or_create_channel_destination_conflict(async_client, test_ses
|
|||
test_session.add(channel)
|
||||
test_session.commit()
|
||||
|
||||
# Attempting to create a new channel at the same directory should fail.
|
||||
with pytest.raises(UnrecoverableDownloadError):
|
||||
get_or_create_channel(test_session, source_id='new_id', name='New Channel',
|
||||
url='https://example.com', destination=existing_dir)
|
||||
# Destination matches an existing channel, so it should be returned even though source_id/name/url don't match.
|
||||
result = get_or_create_channel(test_session, source_id='new_id', name='New Channel',
|
||||
url='https://example.com', destination=existing_dir)
|
||||
assert result.id == channel.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_or_create_channel_destination_nested_conflict(async_client, test_session, test_directory):
|
||||
"""Creating a channel at a destination nested inside an existing channel's directory should raise
|
||||
UnrecoverableDownloadError."""
|
||||
"""When destination is nested inside an existing channel's directory, the parent channel should be returned."""
|
||||
from wrolpi.collections import Collection
|
||||
|
||||
parent_dir = test_directory / 'videos/parent_channel'
|
||||
|
|
@ -357,11 +356,11 @@ async def test_get_or_create_channel_destination_nested_conflict(async_client, t
|
|||
test_session.add(channel)
|
||||
test_session.commit()
|
||||
|
||||
# Nested directory should conflict.
|
||||
# Nested directory should find the parent channel.
|
||||
nested_dir = parent_dir / 'nested'
|
||||
with pytest.raises(UnrecoverableDownloadError):
|
||||
get_or_create_channel(test_session, source_id='nested_id', name='Nested Channel',
|
||||
url='https://example.net', destination=nested_dir)
|
||||
result = get_or_create_channel(test_session, source_id='nested_id', name='Nested Channel',
|
||||
url='https://example.net', destination=nested_dir)
|
||||
assert result.id == channel.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -386,6 +385,48 @@ async def test_get_or_create_channel_existing_ignores_destination(async_client,
|
|||
assert result.directory == existing_dir
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_channel_prefers_destination_over_ytdlp(async_client, test_session, test_directory, channel_factory):
|
||||
"""When yt-dlp metadata identifies ChannelA but the download destination is ChannelB's directory,
|
||||
_get_channel should return ChannelB (the destination's channel)."""
|
||||
channel_a = channel_factory(source_id='channel_a_id', name='ChannelA',
|
||||
url='https://example.com/channel_a')
|
||||
channel_b = channel_factory(source_id='channel_b_id', name='ChannelB',
|
||||
url='https://example.com/channel_b')
|
||||
|
||||
# Download has yt-dlp metadata matching ChannelA, but destination is ChannelB's directory.
|
||||
download = Download(
|
||||
url='https://example.com/video1',
|
||||
destination=channel_b.directory,
|
||||
info_json={'channel': 'ChannelA', 'channel_id': 'channel_a_id',
|
||||
'channel_url': 'https://example.com/channel_a'},
|
||||
)
|
||||
|
||||
channel = await VideoDownloader._get_channel(test_session, download)
|
||||
|
||||
assert channel.id == channel_b.id, 'Should prefer the destination channel over yt-dlp metadata'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_channel_uses_ytdlp_when_destination_matches(async_client, test_session, test_directory,
|
||||
channel_factory):
|
||||
"""When yt-dlp metadata identifies ChannelA and the download destination is also ChannelA's directory,
|
||||
_get_channel should return ChannelA (no conflict)."""
|
||||
channel_a = channel_factory(source_id='channel_a_id', name='ChannelA',
|
||||
url='https://example.com/channel_a')
|
||||
|
||||
download = Download(
|
||||
url='https://example.com/video1',
|
||||
destination=channel_a.directory,
|
||||
info_json={'channel': 'ChannelA', 'channel_id': 'channel_a_id',
|
||||
'channel_url': 'https://example.com/channel_a'},
|
||||
)
|
||||
|
||||
channel = await VideoDownloader._get_channel(test_session, download)
|
||||
|
||||
assert channel.id == channel_a.id
|
||||
|
||||
|
||||
def test_channel_downloader_hidden(video_download_manager):
|
||||
"""
|
||||
ChannelDownloader should not be presented to the User.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue