Normalizing video URLs in RSSDownloader so that we know which ones have been skipped.

This commit is contained in:
Roland 2026-02-18 11:55:04 -07:00
parent b8d3b5318b
commit 308b1d9c88
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 124 additions and 1 deletions

View file

@ -1565,6 +1565,13 @@ class RSSDownloader(Downloader):
else:
logger.warning(f'RSS entry {idx} did not have a link!')
if download.sub_downloader == 'video':
# Normalize video URLs before checking if already downloaded.
# YouTube Shorts URLs (e.g., youtube.com/shorts/ABC123) must be converted to watch URLs
# (e.g., youtube.com/watch?v=ABC123) to match what is stored in the database.
from modules.videos.normalize_video_url import normalize_video_url
urls = [normalize_video_url(i) for i in urls]
# Only download new URLs.
with get_db_session() as session:
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(session, *urls)]]

View file

@ -5,7 +5,7 @@ import mock
import pytest
from sqlalchemy.orm import Session
from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadManager, RSSDownloader
from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadManager, RSSDownloader, download_manager
class RSSHTTPDownloader(Downloader):
@ -178,3 +178,119 @@ async def test_rss_downloader_uses_destination_column(test_session, test_downloa
'Child download should inherit destination from RSS download.destination column'
assert child_b.settings['destination'].endswith('archive/custom/path'), \
f"Expected destination ending with 'archive/custom/path', got {child_b.settings.get('destination')}"
@pytest.mark.asyncio
async def test_rss_downloader_normalizes_youtube_shorts_urls(test_session, test_download_manager, await_switches,
video_factory):
"""RSS downloader should normalize YouTube Shorts URLs before checking if already downloaded.
YouTube Shorts URLs (youtube.com/shorts/ABC123) are stored as watch URLs (youtube.com/watch?v=ABC123)
in the database after download. The RSS feed may continue to return shorts URLs, so normalization
is needed to recognize that the video was already downloaded.
"""
from modules.videos.downloader import VideoDownloader
from wrolpi.files.models import FileGroup
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
video_downloader = VideoDownloader()
test_download_manager.register_downloader(video_downloader)
# Create a video that was already downloaded. The URL is stored in the normalized watch format
# (as yt-dlp would store it after downloading).
video = video_factory(source_id='ABC123')
video.file_group.url = 'https://www.youtube.com/watch?v=ABC123' # Normalized URL
test_session.commit()
# Verify the video exists with the normalized URL.
file_groups = list(test_session.query(FileGroup).filter(
FileGroup.url == 'https://www.youtube.com/watch?v=ABC123'
))
assert len(file_groups) == 1, 'Video should exist with normalized URL'
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
# RSS feed returns the shorts URL format (same video, different URL format).
mock_parse_feed.return_value = dict(
bozo=0,
entries=[
dict(link='https://www.youtube.com/shorts/ABC123'), # Shorts URL, same video
dict(link='https://www.youtube.com/shorts/DEF456'), # New video, not downloaded
]
)
test_download_manager.create_download(
test_session,
'https://example.com/feed',
rss_downloader.name,
sub_downloader_name='video',
)
await test_download_manager.wait_for_all_downloads()
# Get all downloads.
downloads = test_session.query(Download).order_by(Download.id).all()
# Should have:
# 1. RSS feed download (complete)
# 2. Only the new video (DEF456) should be added for download
# The already downloaded video (ABC123) should NOT be re-added
download_urls = [d.url for d in downloads]
assert 'https://example.com/feed' in download_urls, 'RSS feed download should exist'
# The shorts URL for ABC123 should NOT be in downloads (it was recognized as already downloaded).
assert 'https://www.youtube.com/shorts/ABC123' not in download_urls, \
'Already downloaded video (shorts URL) should not be re-added to downloads'
assert 'https://www.youtube.com/watch?v=ABC123' not in download_urls, \
'Already downloaded video (normalized URL) should not be re-added to downloads'
# The new video (DEF456) should be added for download with the normalized URL.
assert 'https://www.youtube.com/watch?v=DEF456' in download_urls, \
'New video should be added to downloads with normalized URL'
@pytest.mark.asyncio
async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_session, test_download_manager, await_switches):
"""RSS downloader should normalize YouTube Shorts URLs before checking the skip list.
If a video was skipped with a normalized URL, the RSS feed returning a shorts URL should
still recognize it as skipped.
"""
from modules.videos.downloader import VideoDownloader
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
video_downloader = VideoDownloader()
test_download_manager.register_downloader(video_downloader)
# Add a video to the skip list using the normalized URL format.
download_manager.add_to_skip_list('https://www.youtube.com/watch?v=SKIPPED123')
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
# RSS feed returns the shorts URL format (same video, different URL format).
mock_parse_feed.return_value = dict(
bozo=0,
entries=[
dict(link='https://www.youtube.com/shorts/SKIPPED123'), # Shorts URL, should be skipped
dict(link='https://www.youtube.com/shorts/NEW456'), # New video, should be downloaded
]
)
test_download_manager.create_download(
test_session,
'https://example.com/feed',
rss_downloader.name,
sub_downloader_name='video',
)
await test_download_manager.wait_for_all_downloads()
# Get all downloads.
downloads = test_session.query(Download).order_by(Download.id).all()
download_urls = [d.url for d in downloads]
# The skipped video should NOT be in downloads.
assert 'https://www.youtube.com/shorts/SKIPPED123' not in download_urls, \
'Skipped video (shorts URL) should not be added to downloads'
assert 'https://www.youtube.com/watch?v=SKIPPED123' not in download_urls, \
'Skipped video (normalized URL) should not be added to downloads'
# The new video should be added for download with the normalized URL.
assert 'https://www.youtube.com/watch?v=NEW456' in download_urls, \
'New video should be added to downloads with normalized URL'