mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Channels are linked to downloads by directory or download source id when downloading, or when importing config.
This commit is contained in:
parent
a5dee64af2
commit
3f5470d974
4 changed files with 127 additions and 4 deletions
|
|
@ -240,6 +240,12 @@ class ChannelDownloader(Downloader, ABC):
|
|||
channel = get_or_create_channel(session, channel_source_id, download.url, name, channel_tag_name)
|
||||
channel.dict() # get all attributes while we have the session.
|
||||
|
||||
# Link this channel download to the channel's collection.
|
||||
if channel and channel.collection_id and not download.collection_id:
|
||||
download.collection_id = channel.collection_id
|
||||
if session:
|
||||
session.commit()
|
||||
|
||||
location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None
|
||||
|
||||
# The settings to send to the VideoDownloader.
|
||||
|
|
|
|||
|
|
@ -703,19 +703,26 @@ def link_channel_and_downloads(session: Session, channel_: Type[Base] = Channel,
|
|||
Downloads are linked to Collections (via collection_id) rather than directly to Channels.
|
||||
This function finds Channels and links their Downloads to the Channel's Collection.
|
||||
"""
|
||||
|
||||
def get_destination(d):
|
||||
"""Get destination from download.destination column or settings['destination']."""
|
||||
if d.destination:
|
||||
return str(d.destination)
|
||||
return (d.settings or dict()).get('destination')
|
||||
|
||||
# Only Downloads with a frequency can be a Collection Download.
|
||||
downloads = list(session.query(download_).filter(download_.frequency.isnot(None)).all())
|
||||
# Download.url is unique and cannot be null.
|
||||
downloads_by_url = {i.url: i for i in downloads}
|
||||
# Many Downloads may share the same destination.
|
||||
downloads_with_destination = [i for i in downloads if (i.settings or dict()).get('destination')]
|
||||
# Many Downloads may share the same destination (from either column or settings).
|
||||
downloads_with_destination = [(d, get_destination(d)) for d in downloads if get_destination(d)]
|
||||
channels = session.query(channel_).all()
|
||||
|
||||
need_commit = False
|
||||
for channel in channels:
|
||||
directory = str(channel.directory)
|
||||
for download in downloads_with_destination:
|
||||
if download.settings['destination'] == directory and not download.collection_id:
|
||||
for download, destination in downloads_with_destination:
|
||||
if destination == directory and not download.collection_id:
|
||||
download.collection_id = channel.collection_id
|
||||
need_commit = True
|
||||
|
||||
|
|
@ -738,6 +745,18 @@ def link_channel_and_downloads(session: Session, channel_: Type[Base] = Channel,
|
|||
download.collection_id = channel.collection_id
|
||||
need_commit = True
|
||||
|
||||
# Associate any Download which has info_json channel_id matching a Channel's source_id.
|
||||
for download in downloads:
|
||||
if download.collection_id:
|
||||
continue
|
||||
info_json = download.info_json or {}
|
||||
channel_source_id = info_json.get('channel_id') or info_json.get('id')
|
||||
if channel_source_id:
|
||||
channel = channel_.get_by_source_id(session, channel_source_id)
|
||||
if channel:
|
||||
download.collection_id = channel.collection_id
|
||||
need_commit = True
|
||||
|
||||
if need_commit:
|
||||
session.commit()
|
||||
|
||||
|
|
|
|||
|
|
@ -706,3 +706,45 @@ async def test_video_download_always_use_cookies(test_session, test_directory, m
|
|||
|
||||
assert '--cookies-from-browser' in cmd
|
||||
assert 'chromium:some directory' in cmd
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_downloader_sets_collection_id(test_session, channel_factory, mock_video_extract_info,
|
||||
video_download_manager, await_switches):
|
||||
"""Test that ChannelDownloader.do_download sets download.collection_id to the channel's collection.
|
||||
|
||||
When a channel download runs, it should link the download to the channel's collection immediately.
|
||||
"""
|
||||
channel = channel_factory(url='https://example.com/channel', source_id='UCtest123')
|
||||
|
||||
download = Download(
|
||||
url='https://example.com/channel',
|
||||
downloader=channel_downloader.name,
|
||||
frequency=99,
|
||||
)
|
||||
test_session.add(download)
|
||||
test_session.commit()
|
||||
|
||||
assert not download.collection_id, 'Download should not be linked yet'
|
||||
|
||||
# Mock extract_info to return channel info
|
||||
mock_info = {
|
||||
'id': 'UCtest123',
|
||||
'channel_id': 'UCtest123',
|
||||
'uploader': channel.name,
|
||||
'entries': [],
|
||||
'webpage_url_basename': 'videos',
|
||||
}
|
||||
mock_video_extract_info.return_value = mock_info
|
||||
|
||||
# Mock Channel.refresh_files to avoid background tasks
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \
|
||||
mock.patch.object(Channel, 'refresh_files'):
|
||||
mock_video_do_download.return_value = DownloadResult(success=True)
|
||||
await channel_downloader.do_download(download)
|
||||
|
||||
await await_switches()
|
||||
|
||||
test_session.refresh(download)
|
||||
assert download.collection_id == channel.collection_id, \
|
||||
'ChannelDownloader should link download to channel collection'
|
||||
|
|
|
|||
|
|
@ -250,6 +250,62 @@ def test_link_channel_and_downloads_migration(async_client, test_session, channe
|
|||
assert not d2c.collection_id
|
||||
|
||||
|
||||
def test_link_channel_and_downloads_destination_column(test_session, channel_factory, test_download_manager):
|
||||
"""Test that link_channel_and_downloads matches by download.destination column, not just settings['destination'].
|
||||
|
||||
This simulates what happens when downloads are imported from config - the destination column is set directly,
|
||||
not via settings['destination'].
|
||||
"""
|
||||
channel = channel_factory()
|
||||
|
||||
# Create download with destination COLUMN set (like import_config does)
|
||||
# NOT using settings={'destination': ...}
|
||||
download = Download(
|
||||
url='https://example.com/channel/videos',
|
||||
downloader=ChannelDownloader.name,
|
||||
frequency=99,
|
||||
destination=channel.directory, # Using column directly
|
||||
)
|
||||
test_session.add(download)
|
||||
test_session.commit()
|
||||
|
||||
assert not download.collection_id, 'Download should not be linked yet'
|
||||
|
||||
lib.link_channel_and_downloads(test_session)
|
||||
|
||||
test_session.refresh(download)
|
||||
assert download.collection_id == channel.collection_id, \
|
||||
'Download should be linked to channel via destination column'
|
||||
|
||||
|
||||
def test_link_channel_and_downloads_by_source_id(test_session, channel_factory, test_download_manager):
|
||||
"""Test that link_channel_and_downloads matches by info_json channel_id to Channel.source_id.
|
||||
|
||||
When ChannelDownloader runs, it populates download.info_json with channel_id. This should match
|
||||
against Channel.source_id even if the URLs don't match.
|
||||
"""
|
||||
channel = channel_factory(source_id='UC123456789')
|
||||
|
||||
# Create download with info_json containing channel_id (like ChannelDownloader sets)
|
||||
# Note: different URL than channel URL, should still match by source_id
|
||||
download = Download(
|
||||
url='https://example.com/different-channel-url',
|
||||
downloader=ChannelDownloader.name,
|
||||
frequency=99,
|
||||
info_json={'channel_id': 'UC123456789', 'id': 'UC123456789', 'uploader': 'Test'},
|
||||
)
|
||||
test_session.add(download)
|
||||
test_session.commit()
|
||||
|
||||
assert not download.collection_id, 'Download should not be linked yet'
|
||||
|
||||
lib.link_channel_and_downloads(test_session)
|
||||
|
||||
test_session.refresh(download)
|
||||
assert download.collection_id == channel.collection_id, \
|
||||
'Download should be linked to channel via source_id matching'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_format_videos_destination(async_client, test_directory):
|
||||
"""Videos destination is formatted according to the WROLPiConfig."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue