- | Video File |
+ {isAudio ? 'Audio File' : 'Video File'} |
{video['video_path']
- ? <>
+ ? <>
{video['video_path']}
{fileSize(videoPathFile)}
: 'Unknown'}
@@ -434,6 +454,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
}
let videoSource = ;
+ const isAudio = videoFile.mimetype && videoFile.mimetype.startsWith('audio/');
let prevNextVideosSegment = No related videos found. ;
if (prevFile || nextFile) {
@@ -463,7 +484,42 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
- }
diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py
index 3318dade..e41904e9 100644
--- a/modules/videos/__init__.py
+++ b/modules/videos/__init__.py
@@ -1,6 +1,7 @@
import asyncio
from typing import Callable, List, Tuple
+from sqlalchemy import or_
from sqlalchemy.orm import Session
from modules.videos.models import Video
@@ -24,7 +25,7 @@ async def video_modeler(progress_callback: Callable[[int], None] = None):
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup, Video).filter(
FileGroup.indexed != True,
- FileGroup.mimetype.like('video/%'),
+ or_(FileGroup.mimetype.like('video/%'), FileGroup.mimetype.like('audio/%')),
).outerjoin(Video, Video.file_group_id == FileGroup.id) \
.limit(VIDEO_PROCESSING_LIMIT)
file_groups: List[Tuple[FileGroup, Video]] = list(file_groups)
@@ -79,10 +80,14 @@ async def video_modeler(progress_callback: Callable[[int], None] = None):
def video_cleanup():
logger.info('Claiming Videos for their Channels')
with get_db_curs(commit=True) as curs:
- # Delete all Videos if the FileModel no longer contains a video.
+ # Delete all Videos if the FileModel no longer contains a video or audio file.
curs.execute('''
WITH deleted AS
- (UPDATE file_group SET model = null WHERE model = 'video' AND mimetype NOT LIKE 'video/%' RETURNING id)
+ (UPDATE file_group SET model = null
+ WHERE model = 'video'
+ AND mimetype NOT LIKE 'video/%%'
+ AND mimetype NOT LIKE 'audio/%%'
+ RETURNING id)
DELETE
FROM video
WHERE file_group_id = ANY (select id from deleted)
diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py
index 27eec02c..2a0774ea 100644
--- a/modules/videos/conftest.py
+++ b/modules/videos/conftest.py
@@ -192,6 +192,55 @@ def video_factory(test_session, test_directory):
return factory
+@pytest.fixture
+def audio_factory(test_session, test_directory):
+ """Creates audio-only Video records for testing."""
+
+ def factory(channel_id: int = None, title: str = None, upload_date=None,
+ with_info_json: dict = None, with_poster_ext: str = None,
+ source_id: str = None, tag_names: List[str] = None) -> Video:
+ title = title or str(uuid4())
+ tag_names = tag_names or list()
+
+ if channel_id:
+ channel = Channel.find_by_id(test_session, channel_id)
+ path = (channel.directory or test_directory) / f'{title}.mp3'
+ path.parent.mkdir(exist_ok=True, parents=True)
+ else:
+ (test_directory / 'videos/NO CHANNEL').mkdir(exist_ok=True, parents=True)
+ path = test_directory / f'videos/NO CHANNEL/{title}.mp3'
+
+ assert str(path).startswith(str(test_directory)), 'Audio must be in the test directory'
+
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny.mp3', path)
+
+ info_json_path = None
+ if with_info_json:
+ with_info_json = {'duration': 5, 'epoch': 12345678} if with_info_json is True else with_info_json
+ info_json_path = path.with_suffix('.info.json')
+ info_json_path.write_text(json.dumps(with_info_json))
+
+ poster_path = None
+ if with_poster_ext:
+ poster_path = path.with_suffix(f'.{with_poster_ext.lstrip(".")}')
+ Image.new('RGB', (25, 25)).save(poster_path)
+
+ paths = list(filter(None, (path, info_json_path, poster_path)))
+
+ video = Video.from_paths(test_session, *paths)
+ video.channel_id = channel_id
+ video.source_id = source_id or title
+ video.file_group.published_datetime = upload_date
+ video.validate(test_session)
+
+ for tag_name in tag_names:
+ video.add_tag(test_session, tag_name)
+
+ return video
+
+ return factory
+
+
@pytest.fixture
def video_download_manager(test_download_manager) -> DownloadManager:
"""Get a DownloadManager ready to download Videos and Channels."""
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index f7ae73d4..9d1c8770 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -91,6 +91,17 @@ def _log_ytdlp_versions():
logger.info(f'yt-dlp={ytdlp_version} yt-dlp-ejs={ejs_version}')
+# yt-dlp codec names that differ from their output file extension.
+AUDIO_CODEC_EXTENSION_MAP = {
+ 'vorbis': 'ogg',
+}
+
+
+def audio_output_extension(audio_format: str) -> str:
+ """Map a yt-dlp audio codec name to the file extension it actually produces."""
+ return AUDIO_CODEC_EXTENSION_MAP.get(audio_format, audio_format)
+
+
VIDEO_RESOLUTION_MAP = {
'360p': [
'134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360',
@@ -639,6 +650,8 @@ class VideoDownloader(Downloader, ABC):
logger.debug(f'Downloading {url} with {tag_names=}')
effective = get_effective_video_settings(settings)
+ audio_only = effective.get('audio_only', False)
+ audio_format = effective.get('audio_format', 'mp3')
logs = None # noqa
try:
@@ -647,29 +660,34 @@ class VideoDownloader(Downloader, ABC):
video_resolutions_str = ','.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i])
video_format = effective['video_format']
- video_path, entry = self.prepare_filename(url, out_dir, video_resolutions_str, video_format)
+ video_path, entry = self.prepare_filename(url, out_dir, video_resolutions_str, video_format,
+ audio_only=audio_only, audio_format=audio_format)
if settings.get('download_metadata_only'):
# User has requested refresh of video metadata, skip the rest of the video downloader.
return await self.download_info_json(download, video_path)
+ # Common yt-dlp flags shared by both video and audio-only downloads.
cmd = (
str(YT_DLP_BIN),
- '-f', video_resolutions_str,
- '--match-filter', '!is_live', # Do not attempt to download Live videos.
+ '--match-filter', '!is_live',
'--sub-format', DEFAULT_CAPTION_FORMAT,
'--convert-subs', DEFAULT_CAPTION_FORMAT,
'--convert-thumbnails', DEFAULT_POSTER_FORMAT,
- '--merge-output-format', video_format,
'--no-cache-dir',
'--newline',
'--compat-options', 'no-live-chat',
- # Get top 20 comments, 10 replies per parent.
'--write-comments',
'--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
- # Use experimental feature to merge files.
- '--ppa', 'Merger+ffmpeg_o1:-strict -2',
)
+ if audio_only:
+ cmd = (*cmd, '-f', 'bestaudio', '-x', '--audio-format', audio_format)
+ else:
+ cmd = (*cmd,
+ '-f', video_resolutions_str,
+ '--merge-output-format', video_format,
+ '--ppa', 'Merger+ffmpeg_o1:-strict -2',
+ )
if effective['continue_dl']:
cmd = (*cmd, '--continue')
if effective['writesubtitles']:
@@ -717,19 +735,20 @@ class VideoDownloader(Downloader, ABC):
location=location,
)
- preferred_path = video_path.with_suffix(f'.{video_format}')
+ output_format = audio_output_extension(audio_format) if audio_only else video_format
+ preferred_path = video_path.with_suffix(f'.{output_format}')
if not video_path.is_file() and preferred_path.is_file():
- # Prepared filename does not exist, but video with preferred video extension does, it was probably
- # remuxed by yt-dlp.
+ # Prepared filename does not exist, but file with preferred extension does, it was probably
+ # remuxed/converted by yt-dlp.
video_path = preferred_path
- logger.info(f'Using preferred video file which exists: {preferred_path}')
+ logger.info(f'Using preferred file which exists: {preferred_path}')
if not video_path.is_file():
# yt-dlp may have appended the real extension to a bogus one (e.g. .NA.mp4 from a playlist entry).
- appended_path = video_path.parent / f'{video_path.name}.{video_format}'
+ appended_path = video_path.parent / f'{video_path.name}.{output_format}'
if appended_path.is_file():
video_path = appended_path
- logger.info(f'Using video file with appended extension: {appended_path}')
+ logger.info(f'Using file with appended extension: {appended_path}')
if video_path.suffix == '.part':
return DownloadResult(
@@ -754,7 +773,7 @@ class VideoDownloader(Downloader, ABC):
retry_seconds=retry_seconds,
)
- if not ffmpeg_video_complete(video_path):
+ if not audio_only and not ffmpeg_video_complete(video_path):
return DownloadResult(
success=False,
error='Video was incomplete',
@@ -806,8 +825,8 @@ class VideoDownloader(Downloader, ABC):
await video.get_ffprobe_json()
- # Check that video has both audio and video streams.
- if not video.get_streams_by_codec_type('video'):
+ # Check that the download has the expected streams.
+ if not audio_only and not video.get_streams_by_codec_type('video'):
return DownloadResult(
success=False,
error='Video was downloaded but did not contain video stream',
@@ -816,7 +835,7 @@ class VideoDownloader(Downloader, ABC):
if not video.get_streams_by_codec_type('audio'):
return DownloadResult(
success=False,
- error='Video was downloaded but did not contain audio stream',
+ error='Download did not contain audio stream',
location=location,
)
logger.info(f'Successfully downloaded video {url} {video}')
@@ -918,34 +937,48 @@ class VideoDownloader(Downloader, ABC):
return None
@staticmethod
- def prepare_filename(url: str, out_dir: pathlib.Path, video_resolutions: str, video_format: str) \
+ def prepare_filename(url: str, out_dir: pathlib.Path, video_resolutions: str, video_format: str,
+ audio_only: bool = False, audio_format: str = 'mp3') \
-> Tuple[pathlib.Path, dict]:
- """Get the full path of a video file from its URL using yt-dlp."""
+ """Get the full path of a video/audio file from its URL using yt-dlp."""
if not out_dir.is_dir():
raise ValueError(f'Output directory does not exist! {out_dir=}')
- # YoutubeDL expects specific options, add onto the default options
+ # YoutubeDL expects specific options, add onto the default options.
+ # Deep copy to avoid mutating the shared singleton config dict.
config = get_videos_downloader_config()
- options = config.yt_dlp_options
+ options = copy.deepcopy(config.yt_dlp_options)
+
+ if audio_only:
+ options['format'] = 'bestaudio'
+ options['postprocessors'] = [{
+ 'key': 'FFmpegExtractAudio',
+ 'preferredcodec': audio_format,
+ }]
+ # merge_output_format is not used for audio-only.
+ options.pop('merge_output_format', None)
+ else:
+ options['merge_output_format'] = video_format
+ options['format'] = video_resolutions
+
# Convert WROLPi-specific variables to yt-dlp syntax
converted_format = convert_wrolpi_filename_format(config.file_name_format)
# yt-dlp expects the absolute path.
options['outtmpl'] = f'{out_dir}/{converted_format}'
- options['merge_output_format'] = video_format
- # options['remuxvideo'] = video_format
- options['format'] = video_resolutions
options['cachdir'] = YTDLP_CACHE_DIR
+ output_ext = audio_output_extension(audio_format) if audio_only else video_format
+
# Create a new YoutubeDL for the output directory.
ydl = YoutubeDL(copy.deepcopy(options))
ydl.params['logger'] = ydl_logger
ydl.add_default_info_extractors()
- # Get the path where the video will be saved.
+ # Get the path where the file will be saved.
try:
entry = extract_info(url, ydl=ydl, process=True)
# If the entry is a playlist (e.g. Twitter/X post with multiple videos),
- # use the first video entry for filename preparation.
+ # use the first entry for filename preparation.
if entry.get('entries'):
entries = list(entry['entries']) if not isinstance(entry['entries'], list) else entry['entries']
if entries:
@@ -967,11 +1000,11 @@ class VideoDownloader(Downloader, ABC):
parent = full_path.parent
filename, _ = split_path_stem_and_suffix(full_path.name)
- # Trim long filename, add video suffix, add back into parent directory.
+ # Trim long filename, add output suffix, add back into parent directory.
filename = escape_file_name(filename)
- final_filename = parent / f'{filename}.{video_format}'
+ final_filename = parent / f'{filename}.{output_ext}'
final_filename = trim_file_name(final_filename)
- logger.debug(f'Video file name was too long. Trimmed to: {final_filename.name}')
+ logger.debug(f'File name was too long. Trimmed to: {final_filename.name}')
# Get entry info json.
options['outtmpl'] = str(final_filename)
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 1d97e9cf..06b4cad0 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -144,7 +144,8 @@ def validate_video(session: Session, video: Video, channel_generate_poster: bool
video.file_group.published_datetime = json_published_datetime or published_date
video.source_id = video.source_id or source_id
- if channel_generate_poster:
+ is_audio = video.file_group.mimetype and video.file_group.mimetype.startswith('audio/')
+ if channel_generate_poster and not is_audio:
# Try to convert/generate, but keep the old poster if those fail.
new_poster_path, duration = convert_or_generate_poster(video) or video.poster_path
if new_poster_path:
@@ -565,6 +566,8 @@ class VideoDownloaderConfigValidator:
download_missing_video_comments: bool = False
user_agent: str = ''
sleep_requests: float = 0.75
+ audio_only: bool = False
+ audio_format: str = 'mp3'
def __post_init__(self):
allowed_fields = {i.name for i in dataclasses.fields(VideoDownloaderConfigYtDlpOptionsValidator)}
@@ -594,6 +597,8 @@ class VideoDownloaderConfig(ConfigFile):
download_missing_video_comments=False,
user_agent='',
sleep_requests=0.75,
+ audio_only=False,
+ audio_format='mp3',
)
validator = VideoDownloaderConfigValidator
@@ -690,6 +695,22 @@ class VideoDownloaderConfig(ConfigFile):
def sleep_requests(self, value: float):
self.update({**self._config, 'sleep_requests': value})
+ @property
+ def audio_only(self) -> bool:
+ return self._config.get('audio_only', False)
+
+ @audio_only.setter
+ def audio_only(self, value: bool):
+ self.update({**self._config, 'audio_only': value})
+
+ @property
+ def audio_format(self) -> str:
+ return self._config.get('audio_format', 'mp3')
+
+ @audio_format.setter
+ def audio_format(self, value: str):
+ self.update({**self._config, 'audio_format': value})
+
def import_config(self, file: pathlib.Path = None, send_events=False):
super().import_config(file, send_events)
self.successful_import = True
@@ -718,7 +739,7 @@ def set_test_downloader_config(enabled: bool):
INHERITABLE_VIDEO_SETTINGS = {
'video_resolutions', 'video_format', 'writesubtitles', 'writeautomaticsub',
'writethumbnail', 'writeinfojson', 'yt_dlp_extra_args', 'sleep_requests',
- 'user_agent', 'continue_dl', 'nooverwrites',
+ 'user_agent', 'continue_dl', 'nooverwrites', 'audio_only', 'audio_format',
}
# Subset of inheritable settings allowed at channel/RSS level.
@@ -740,6 +761,8 @@ def _extract_global_inheritable(config: VideoDownloaderConfig) -> dict:
'user_agent': config.user_agent,
'continue_dl': config.continue_dl,
'nooverwrites': config.nooverwrites,
+ 'audio_only': config.audio_only,
+ 'audio_format': config.audio_format,
}
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 5991c16e..2371074e 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -301,6 +301,10 @@ class Video(ModelHelper, Base):
for file_group in self.file_group.my_video_files():
return file_group['path']
+ # Or the first audio file (for audio-only downloads).
+ for file_group in self.file_group.my_audio_files():
+ return file_group['path']
+
@property
def poster_file(self) -> Optional[dict]:
for file_group in self.file_group.my_poster_files():
diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py
index 145304b1..e88ca711 100644
--- a/modules/videos/test/test_api.py
+++ b/modules/videos/test/test_api.py
@@ -264,6 +264,40 @@ async def test_search_videos(test_session, video_factory, assert_video_search, s
await assert_video_search(order_by=order_by)
+@pytest.mark.asyncio
+async def test_search_audio_only(test_session, audio_factory, assert_video_search):
+ """Audio-only files should be searchable alongside videos."""
+ audio1 = audio_factory(title='podcast episode one', upload_date='2025-01-01', with_info_json=True)
+ audio2 = audio_factory(title='podcast episode two', upload_date='2025-01-02', with_info_json=True)
+ test_session.commit()
+
+ assert audio1.file_group.mimetype == 'audio/mpeg'
+ assert audio1.file_group.model == 'video'
+
+ # Both audio files are returned.
+ await assert_video_search(assert_total=2, order_by='published_datetime')
+
+ # Search by title.
+ await assert_video_search(search_str='podcast', assert_total=2)
+ await assert_video_search(search_str='episode one', assert_total=1, assert_ids=[audio1.id])
+
+
+@pytest.mark.asyncio
+async def test_search_mixed_video_and_audio(test_session, video_factory, audio_factory, assert_video_search):
+ """Audio and video files should both appear in search results."""
+ video = video_factory(title='tutorial video', upload_date='2025-01-01', with_video_file=True)
+ audio = audio_factory(title='tutorial audio', upload_date='2025-01-02', with_info_json=True)
+ test_session.commit()
+
+ # Both appear when searching.
+ await assert_video_search(assert_total=2, order_by='published_datetime')
+ await assert_video_search(search_str='tutorial', assert_total=2)
+
+ # Each can be found individually.
+ await assert_video_search(search_str='video', assert_total=1, assert_ids=[video.id])
+ await assert_video_search(search_str='audio', assert_total=1, assert_ids=[audio.id])
+
+
@pytest.mark.asyncio
async def test_delete_video(async_client, test_session, video_factory, test_download_manager):
"""When a Video record is deleted, the FileGroup should be deleted. The URL is added to the
diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py
index f479729e..00d50cf7 100644
--- a/modules/videos/video/lib.py
+++ b/modules/videos/video/lib.py
@@ -95,8 +95,8 @@ def search_videos(
headline: bool = False,
) -> Tuple[List[dict], int]:
tag_names = tag_names or []
- # Only search videos.
- wheres = ["fg.mimetype LIKE 'video/%%'"]
+ # Search videos and audio-only files.
+ wheres = ["(fg.mimetype LIKE 'video/%%' OR fg.mimetype LIKE 'audio/%%')"]
joins = list()
join_video = False
diff --git a/test/big_buck_bunny.mp3 b/test/big_buck_bunny.mp3
new file mode 100644
index 00000000..dee502f9
Binary files /dev/null and b/test/big_buck_bunny.mp3 differ
diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py
index b512ba96..b2583443 100644
--- a/wrolpi/files/lib.py
+++ b/wrolpi/files/lib.py
@@ -537,6 +537,8 @@ def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]])
return file
if mimetype.startswith('video/'):
return file
+ if mimetype.startswith('audio/'):
+ return file
if mimetype.startswith('application/epub'):
return file
if mimetype.startswith('application/pdf') and not has_epub:
diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py
index 1285ff8b..9cb93686 100644
--- a/wrolpi/files/models.py
+++ b/wrolpi/files/models.py
@@ -326,6 +326,10 @@ class FileGroup(ModelHelper, Base):
"""Return all my Files that are videos."""
return self.my_files('video/')
+ def my_audio_files(self):
+ """Return all my Files that are audio."""
+ return self.my_files('audio/')
+
def my_json_files(self):
"""Return all my Files that are JSON."""
return self.my_files('application/json')
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index 26da03aa..a60f954b 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -252,6 +252,8 @@ class DownloadSettings:
user_agent: Optional[str] = None
continue_dl: Optional[bool] = None
nooverwrites: Optional[bool] = None
+ audio_only: Optional[bool] = None
+ audio_format: Optional[str] = None
def __post_init__(self):
if self.excluded_urls and self.excluded_urls.endswith(','):
@@ -269,8 +271,11 @@ class DownloadSettings:
if self.download_order not in (None, 'newest', 'oldest', 'views'):
raise ValidationError(f'Download order must be one of newest, oldest, views, or null.')
- if self.video_format not in (None, 'mp4', 'mkv'):
- raise ValidationError(f'Download order must be one of mp4, mkv, or null.')
+ if self.video_format not in (None, 'mp4', 'mkv', 'ogg', 'webm'):
+ raise ValidationError(f'Video format must be one of mp4, mkv, ogg, webm, or null.')
+
+ if self.audio_format not in (None, 'mp3', 'opus', 'm4a', 'flac', 'vorbis', 'wav'):
+ raise ValidationError(f'Audio format must be one of mp3, opus, m4a, flac, vorbis, wav, or null.')
valid_resolutions = ('360p', '480p', '720p', '1080p', '1440p', '2160p', 'maximum')
if not all(i in valid_resolutions for i in self.video_resolutions):
|