mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
Normalizing video URLs in RSSDownloader so that we know which ones have been skipped.
This commit is contained in:
parent
b8d3b5318b
commit
308b1d9c88
2 changed files with 124 additions and 1 deletions
|
|
@ -1565,6 +1565,13 @@ class RSSDownloader(Downloader):
|
|||
else:
|
||||
logger.warning(f'RSS entry {idx} did not have a link!')
|
||||
|
||||
if download.sub_downloader == 'video':
|
||||
# Normalize video URLs before checking if already downloaded.
|
||||
# YouTube Shorts URLs (e.g., youtube.com/shorts/ABC123) must be converted to watch URLs
|
||||
# (e.g., youtube.com/watch?v=ABC123) to match what is stored in the database.
|
||||
from modules.videos.normalize_video_url import normalize_video_url
|
||||
urls = [normalize_video_url(i) for i in urls]
|
||||
|
||||
# Only download new URLs.
|
||||
with get_db_session() as session:
|
||||
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(session, *urls)]]
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import mock
|
|||
import pytest
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadManager, RSSDownloader
|
||||
from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadManager, RSSDownloader, download_manager
|
||||
|
||||
|
||||
class RSSHTTPDownloader(Downloader):
|
||||
|
|
@ -178,3 +178,119 @@ async def test_rss_downloader_uses_destination_column(test_session, test_downloa
|
|||
'Child download should inherit destination from RSS download.destination column'
|
||||
assert child_b.settings['destination'].endswith('archive/custom/path'), \
|
||||
f"Expected destination ending with 'archive/custom/path', got {child_b.settings.get('destination')}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rss_downloader_normalizes_youtube_shorts_urls(test_session, test_download_manager, await_switches,
|
||||
video_factory):
|
||||
"""RSS downloader should normalize YouTube Shorts URLs before checking if already downloaded.
|
||||
|
||||
YouTube Shorts URLs (youtube.com/shorts/ABC123) are stored as watch URLs (youtube.com/watch?v=ABC123)
|
||||
in the database after download. The RSS feed may continue to return shorts URLs, so normalization
|
||||
is needed to recognize that the video was already downloaded.
|
||||
"""
|
||||
from modules.videos.downloader import VideoDownloader
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
video_downloader = VideoDownloader()
|
||||
test_download_manager.register_downloader(video_downloader)
|
||||
|
||||
# Create a video that was already downloaded. The URL is stored in the normalized watch format
|
||||
# (as yt-dlp would store it after downloading).
|
||||
video = video_factory(source_id='ABC123')
|
||||
video.file_group.url = 'https://www.youtube.com/watch?v=ABC123' # Normalized URL
|
||||
test_session.commit()
|
||||
|
||||
# Verify the video exists with the normalized URL.
|
||||
file_groups = list(test_session.query(FileGroup).filter(
|
||||
FileGroup.url == 'https://www.youtube.com/watch?v=ABC123'
|
||||
))
|
||||
assert len(file_groups) == 1, 'Video should exist with normalized URL'
|
||||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
# RSS feed returns the shorts URL format (same video, different URL format).
|
||||
mock_parse_feed.return_value = dict(
|
||||
bozo=0,
|
||||
entries=[
|
||||
dict(link='https://www.youtube.com/shorts/ABC123'), # Shorts URL, same video
|
||||
dict(link='https://www.youtube.com/shorts/DEF456'), # New video, not downloaded
|
||||
]
|
||||
)
|
||||
test_download_manager.create_download(
|
||||
test_session,
|
||||
'https://example.com/feed',
|
||||
rss_downloader.name,
|
||||
sub_downloader_name='video',
|
||||
)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
# Get all downloads.
|
||||
downloads = test_session.query(Download).order_by(Download.id).all()
|
||||
|
||||
# Should have:
|
||||
# 1. RSS feed download (complete)
|
||||
# 2. Only the new video (DEF456) should be added for download
|
||||
# The already downloaded video (ABC123) should NOT be re-added
|
||||
download_urls = [d.url for d in downloads]
|
||||
assert 'https://example.com/feed' in download_urls, 'RSS feed download should exist'
|
||||
|
||||
# The shorts URL for ABC123 should NOT be in downloads (it was recognized as already downloaded).
|
||||
assert 'https://www.youtube.com/shorts/ABC123' not in download_urls, \
|
||||
'Already downloaded video (shorts URL) should not be re-added to downloads'
|
||||
assert 'https://www.youtube.com/watch?v=ABC123' not in download_urls, \
|
||||
'Already downloaded video (normalized URL) should not be re-added to downloads'
|
||||
|
||||
# The new video (DEF456) should be added for download with the normalized URL.
|
||||
assert 'https://www.youtube.com/watch?v=DEF456' in download_urls, \
|
||||
'New video should be added to downloads with normalized URL'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_session, test_download_manager, await_switches):
|
||||
"""RSS downloader should normalize YouTube Shorts URLs before checking the skip list.
|
||||
|
||||
If a video was skipped with a normalized URL, the RSS feed returning a shorts URL should
|
||||
still recognize it as skipped.
|
||||
"""
|
||||
from modules.videos.downloader import VideoDownloader
|
||||
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
video_downloader = VideoDownloader()
|
||||
test_download_manager.register_downloader(video_downloader)
|
||||
|
||||
# Add a video to the skip list using the normalized URL format.
|
||||
download_manager.add_to_skip_list('https://www.youtube.com/watch?v=SKIPPED123')
|
||||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
# RSS feed returns the shorts URL format (same video, different URL format).
|
||||
mock_parse_feed.return_value = dict(
|
||||
bozo=0,
|
||||
entries=[
|
||||
dict(link='https://www.youtube.com/shorts/SKIPPED123'), # Shorts URL, should be skipped
|
||||
dict(link='https://www.youtube.com/shorts/NEW456'), # New video, should be downloaded
|
||||
]
|
||||
)
|
||||
test_download_manager.create_download(
|
||||
test_session,
|
||||
'https://example.com/feed',
|
||||
rss_downloader.name,
|
||||
sub_downloader_name='video',
|
||||
)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
# Get all downloads.
|
||||
downloads = test_session.query(Download).order_by(Download.id).all()
|
||||
download_urls = [d.url for d in downloads]
|
||||
|
||||
# The skipped video should NOT be in downloads.
|
||||
assert 'https://www.youtube.com/shorts/SKIPPED123' not in download_urls, \
|
||||
'Skipped video (shorts URL) should not be added to downloads'
|
||||
assert 'https://www.youtube.com/watch?v=SKIPPED123' not in download_urls, \
|
||||
'Skipped video (normalized URL) should not be added to downloads'
|
||||
|
||||
# The new video should be added for download with the normalized URL.
|
||||
assert 'https://www.youtube.com/watch?v=NEW456' in download_urls, \
|
||||
'New video should be added to downloads with normalized URL'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue