diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js
index 4ca82941..2afb4527 100644
--- a/app/src/components/Videos.js
+++ b/app/src/components/Videos.js
@@ -11,6 +11,7 @@ import {
findPosterPath,
humanFileSize,
humanNumber,
+ InfoHeader,
isoDatetimeToAgoPopup,
mimetypeColor,
PageContainer,
@@ -30,6 +31,7 @@ import {
CardHeader,
Container,
Form,
+ FormDropdown,
Image,
PlaceholderHeader,
PlaceholderLine,
@@ -43,7 +45,7 @@ import {FileRowTagIcon, FilesView} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {Button, Card, Header, Loader, Placeholder, Popup, Segment, Statistic, StatisticGroup} from "./Theme";
-import {deleteVideos, fetchVideoDownloaderConfig, postVideoFileFormat, updateVideoDownloaderConfig} from "../api";
+import {deleteVideos, fetchBrowserProfiles, fetchVideoDownloaderConfig, postVideoFileFormat, updateVideoDownloaderConfig} from "../api";
import {Media, QueryContext, ThemeContext} from "../contexts/contexts";
import _ from "lodash";
import {defaultFileOrder, defaultSearchOrder} from "./Vars";
@@ -257,43 +259,76 @@ function VideosSettings() {
writesubtitles: true,
writethumbnail: true,
},
+ always_use_browser_profile: false,
+ yt_dlp_extra_args: '',
+ browser_profile: '',
};
- const fetcher = async () => {
- return await fetchVideoDownloaderConfig();
- }
+ const configSubmitter = async () => {
+ return await updateVideoDownloaderConfig(configForm.formData);
+ };
- const submitter = async () => {
- return await updateVideoDownloaderConfig(form.formData);
- }
-
- const form = useForm({
- fetcher,
- submitter,
+ const configForm = useForm({
+ fetcher: fetchVideoDownloaderConfig,
+ submitter: configSubmitter,
emptyFormData,
});
+ const emptyProfilesOptions = [{key: null, text: 'No profiles found'}];
+ const loadingProfilesOptions = [{key: null, text: 'Loading profiles, please wait...'}];
+ const [browserProfilesOptions, setBrowserProfilesOptions] = useState(loadingProfilesOptions);
+
+ const localFetchBrowserProfiles = async () => {
+ let tempProfiles = [];
+ let localBrowserProfiles = await fetchBrowserProfiles();
+
+ const chromiumProfiles = localBrowserProfiles['chromium_profiles'];
+ for (const value of Object.values(chromiumProfiles)) {
+ tempProfiles.push({value: value, text: value});
+ }
+ const firefoxProfiles = localBrowserProfiles['firefox_profiles'];
+ for (const value of Object.values(firefoxProfiles)) {
+ tempProfiles.push({value: value, text: value});
+ }
+
+ if (_.isEmpty(tempProfiles)) {
+ tempProfiles = emptyProfilesOptions;
+ }
+ console.debug('VideosSettings: Got browser profiles:', tempProfiles);
+ setBrowserProfilesOptions(tempProfiles);
+ };
+
+ React.useEffect(() => {
+ localFetchBrowserProfiles();
+ }, []);
+
+ const alwaysUseBrowserProfileLabel = ;
+
return
- Video Downloader Settings
+
}
+
+export function VideoBrowserCookiesSelector({
+ form,
+ options,
+ name = 'browser_profile',
+ path = 'browser_profile',
+ }) {
+ const [inputProps, inputAttrs] = form.getSelectionProps({name, path});
+
+ const popupContent =
+ Select a browser profile to use for cookies. This is useful for sites that require login to download videos.
+
+
+ MUST BE FROM THE wrolpi USER!
+
;
+
+ return <>
+
+
+
+}
+
function VideosStatistics() {
useTitle('Video Statistics');
diff --git a/app/src/components/admin/Configs.js b/app/src/components/admin/Configs.js
index 8688d02d..8cc73488 100644
--- a/app/src/components/admin/Configs.js
+++ b/app/src/components/admin/Configs.js
@@ -1,12 +1,18 @@
import {APIButton} from "../Common";
-import {Icon, Table} from "../Theme";
+import {Icon, Popup, Table} from "../Theme";
import React from "react";
import {TableBody, TableCell, TableHeader, TableHeaderCell, TableRow} from "semantic-ui-react";
import {TableRowPlaceholder} from "../Placeholder";
import {useNavigate} from "react-router-dom";
function WarningIcon({ok}) {
- return
+ // null is empty, true is valid, false is invalid.
+ const trigger = ;
+ const content = ok === true ? 'Valid and can be imported.' : ok === false ? 'Invalid, can be overwritten.' : 'Empty config, can be overwritten.';
+ return
}
function ConfigTableRow({config, importConfig, saveConfig, fetchConfigs}) {
diff --git a/app/src/hooks/useForm.js b/app/src/hooks/useForm.js
index dac21c8f..373171d2 100644
--- a/app/src/hooks/useForm.js
+++ b/app/src/hooks/useForm.js
@@ -185,7 +185,7 @@ export function useForm({
}
const value = _.get(formData, path); // Get a path, separated by .'s
if (value === undefined) {
- patchFormData(_.set(formData, path, null));
+ patchFormData(_.set(formData, path, type === 'array' ? [] : null));
}
// Allow bypass of `handleInputEvent` which only handles an input event.
const setValue = newValue => {
@@ -426,7 +426,7 @@ export function UrlsTextarea({name = 'urls', required, form}) {
}
-export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big'}) {
+export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big', disabled=false}) {
const [inputProps, inputAttrs] = form.getCustomProps({name, path})
const iconElm = icon ? : null;
@@ -434,7 +434,7 @@ export function ToggleForm({form, name, label, path, icon = null, iconSize = 'bi
label={label}>
{iconElm}
/dev/null 2>&1; then
+ echo "OK: aria2c can be run"
+else
+ echo "FAILED: aria2c cannot be run"
+fi
+
if /opt/wrolpi/venv/bin/yt-dlp -h >/dev/null 2>&1; then
echo "OK: yt-dlp can be run"
else
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 2d88d01a..3b819b0e 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -565,7 +565,7 @@ async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
READABILITY_BIN, singlefile_file.name, shlex.quote(url))
# Docker containers may not have this directory. But, this directory is necessary on RPi.
cwd = '/home/wrolpi' if os.path.isdir('/home/wrolpi') else None
- result = await run_command(cmd, cwd=cwd, timeout=timeout, debug=True)
+ result = await run_command(cmd, cwd=cwd, timeout=timeout)
logger.debug(f'readability for {url} exited with {result.return_code}')
stdout, stderr = result.stdout.decode(), result.stderr.decode()
diff --git a/modules/map/lib.py b/modules/map/lib.py
index 80f4fc55..197ba8d8 100644
--- a/modules/map/lib.py
+++ b/modules/map/lib.py
@@ -193,7 +193,6 @@ async def run_import_command(*paths: Path):
cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', *paths)
result = await run_command(
cmd,
- debug=True,
timeout=0, # No timeout, map importing could take days.
)
diff --git a/modules/videos/api.py b/modules/videos/api.py
index 69778282..994f613c 100644
--- a/modules/videos/api.py
+++ b/modules/videos/api.py
@@ -66,3 +66,9 @@ async def post_file_format(_: Request, body: schema.VideoFileFormatRequest):
error=str(e),
video_file_format=body.video_file_format,
), status=HTTPStatus.BAD_REQUEST)
+
+
+@content_bp.get('/browser-profiles')
+async def get_browser_profiles(_: Request):
+ browser_profiles = lib.get_browser_profiles()
+ return json_response(browser_profiles)
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 18564c27..99dacab5 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -32,7 +32,7 @@ from .channel.lib import create_channel, get_channel
from .common import get_no_channel_directory, update_view_counts_and_censored, \
ffmpeg_video_complete
from .errors import UnknownChannel
-from .lib import get_videos_downloader_config, YDL, ydl_logger, format_videos_destination
+from .lib import get_videos_downloader_config, YDL, ydl_logger, format_videos_destination, browser_profile_to_yt_dlp_arg
from .models import Video, Channel
from .normalize_video_url import normalize_video_url
from .schema import ChannelPostRequest
@@ -45,7 +45,7 @@ VIDEO_RESOLUTION_MAP = {
'480p': ['135+bestaudio', '231+bestaudio', '135+139', 'mp4-480p', 'res:480'],
'720p': [
'298+bestaudio', '232+bestaudio', # 720@60 avc1 + best audio
- '136+bestaudio', '22', 'res:720'],
+ '136+bestaudio', '22', '311+bestaudio', 'res:720'],
'1080p': [
'299+bestaudio', '312+bestaudio', '270+bestaudio', # 1080@60 avc1 + best audio
'137+bestaudio', '614+bestaudio', 'res:1080'],
@@ -328,7 +328,6 @@ class VideoDownloader(Downloader, ABC):
if download.attempts >= 10:
raise UnrecoverableDownloadError('Max download attempts reached')
- download_id = download.id
url = normalize_video_url(download.url)
# Copy settings into Download (they may be from the ChannelDownloader)
@@ -395,19 +394,13 @@ class VideoDownloader(Downloader, ABC):
# User has requested refresh of video metadata, skip the rest of the video downloader.
return await self.download_info_json(download, video_path)
- # Do the real download.
cmd = (
str(YT_DLP_BIN),
- '-c', # Continue downloads
'-f', video_resolutions,
'--match-filter', '!is_live', # Do not attempt to download Live videos.
- '--write-subs',
- '--write-auto-subs',
'--sub-format', DEFAULT_CAPTION_FORMAT,
'--convert-subs', DEFAULT_CAPTION_FORMAT,
'--convert-thumbnails', DEFAULT_POSTER_FORMAT,
- '--write-thumbnail',
- '--write-info-json',
'--merge-output-format', video_format,
# '--remux-video', video_format,
'--no-cache-dir',
@@ -417,10 +410,32 @@ class VideoDownloader(Downloader, ABC):
'--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
# Use experimental feature to merge files.
'--ppa', 'Merger+ffmpeg_o1:-strict -2',
- '-o', video_path,
- url,
)
- result = await self.process_runner(download, cmd, out_dir)
+ if config.continue_dl:
+ cmd = (*cmd, '--continue')
+ if config.writesubtitles:
+ cmd = (*cmd, '--write-subs', '--write-auto-subs')
+ if config.writethumbnail:
+ cmd = (*cmd, '--write-thumbnail')
+ if config.writeinfojson:
+ cmd = (*cmd, '--write-info-json')
+ if config.yt_dlp_extra_args:
+ cmd = (*cmd, *config.yt_dlp_extra_args.split(' '))
+ if config.browser_profile and (config.always_use_browser_profile or settings.get('use_browser_profile')):
+ browser_profile = pathlib.Path(config.browser_profile)
+ if browser_profile.exists():
+ browser_profile = browser_profile_to_yt_dlp_arg(browser_profile)
+ cmd = (*cmd, '--cookies-from-browser', browser_profile)
+ else:
+ logger.error(f'Browser profile {config.browser_profile} does not exist')
+
+ # Add destination and the video's URL to the command.
+ cmd = (*cmd,
+ '-o', video_path,
+ url,
+ )
+ # Do the real download.
+ result = await self.process_runner(download, cmd, out_dir, debug=True)
stdout = result.stdout.decode()
stderr = result.stderr.decode()
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index e5d7756b..3864ef12 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -24,7 +24,7 @@ from wrolpi.errors import UnknownDirectory
from wrolpi.events import Events
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
-from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
+from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST, WROLPI_HOME
from .common import is_valid_poster, convert_image, \
generate_video_poster, ConfigError, \
extract_video_duration
@@ -380,6 +380,9 @@ class VideoDownloaderConfigValidator:
video_resolutions: List[str] = field(default_factory=lambda: ['1080p', '720p', '480p', 'maximum'])
version: int = None
yt_dlp_options: VideoDownloaderConfigYtDlpOptionsValidator = field(default_factory=dict)
+ yt_dlp_extra_args: str = ''
+ always_use_browser_profile: bool = False
+ browser_profile: str = ''
def __post_init__(self):
allowed_fields = {i.name for i in dataclasses.fields(VideoDownloaderConfigYtDlpOptionsValidator)}
@@ -405,6 +408,9 @@ class VideoDownloaderConfig(ConfigFile):
writethumbnail=True,
youtube_include_dash_manifest=False,
),
+ yt_dlp_extra_args='',
+ always_use_browser_profile=False,
+ browser_profile='',
)
validator = VideoDownloaderConfigValidator
@@ -464,6 +470,30 @@ class VideoDownloaderConfig(ConfigFile):
def yt_dlp_options(self, value: dict):
self.update({'yt_dlp_options': value})
+ @property
+ def yt_dlp_extra_args(self) -> str:
+ return self._config['yt_dlp_extra_args']
+
+ @yt_dlp_extra_args.setter
+ def yt_dlp_extra_args(self, value: str):
+ self.update({**self._config, 'yt_dlp_extra_args': value})
+
+ @property
+ def browser_profile(self) -> str:
+ return self._config['browser_profile']
+
+ @browser_profile.setter
+ def browser_profile(self, value: str):
+ self.update({**self._config, 'browser_profile': value})
+
+ @property
+ def always_use_browser_profile(self) -> bool:
+ return self._config['always_use_browser_profile']
+
+ @always_use_browser_profile.setter
+ def always_use_browser_profile(self, value: bool):
+ self.update({**self._config, 'always_use_browser_profile': value})
+
def import_config(self, file: pathlib.Path = None, send_events=False):
super().import_config(file, send_events)
self.successful_import = True
@@ -595,53 +625,48 @@ def get_channel_source_id(url: str) -> str:
async def get_statistics():
with get_db_curs() as curs:
curs.execute('''
- SELECT
- -- total videos
- COUNT(v.id) AS "videos",
- -- total videos downloaded over the past week/month/year
- COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 week') AS "week",
- COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 month') AS "month",
- COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 year') AS "year",
- -- sum of all video lengths in seconds
- COALESCE(SUM(fg.length), 0) AS "sum_duration",
- -- sum of all video file sizes
- COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size",
- -- largest video
- COALESCE(MAX(fg.size), 0) AS "max_size",
- -- Videos may or may not have comments.
- COUNT(v.id) FILTER ( WHERE v.have_comments = TRUE ) AS "have_comments",
- COUNT(v.id) FILTER ( WHERE v.have_comments = FALSE AND v.comments_failed = FALSE
- and fg.censored = false and fg.url is not null) AS "missing_comments",
- COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments",
- COUNT(v.id) FILTER ( WHERE fg.censored = TRUE ) as "censored_videos"
- FROM
- video v
- LEFT JOIN file_group fg on v.file_group_id = fg.id
- ''')
+ SELECT
+ -- total videos
+ COUNT(v.id) AS "videos",
+ -- total videos downloaded over the past week/month/year
+ COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 week') AS "week",
+ COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 month') AS "month",
+ COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 year') AS "year",
+ -- sum of all video lengths in seconds
+ COALESCE(SUM(fg.length), 0) AS "sum_duration",
+ -- sum of all video file sizes
+ COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size",
+ -- largest video
+ COALESCE(MAX(fg.size), 0) AS "max_size",
+ -- Videos may or may not have comments.
+ COUNT(v.id) FILTER ( WHERE v.have_comments = TRUE ) AS "have_comments",
+ COUNT(v.id) FILTER ( WHERE v.have_comments = FALSE AND v.comments_failed = FALSE
+ and fg.censored = false and
+ fg.url is not null) AS "missing_comments",
+ COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments",
+ COUNT(v.id) FILTER ( WHERE fg.censored = TRUE ) as "censored_videos"
+ FROM video v
+ LEFT JOIN file_group fg on v.file_group_id = fg.id
+ ''')
video_stats = dict(curs.fetchone())
# Get the total videos downloaded every month for the past two years.
curs.execute('''
- SELECT
- DATE_TRUNC('month', months.a),
- COUNT(video.id)::BIGINT,
- SUM(size)::BIGINT AS "size"
- FROM
- generate_series(
- date_trunc('month', current_date) - interval '2 years',
- date_trunc('month', current_date) - interval '1 month',
- '1 month'::interval) AS months(a),
- video
- LEFT JOIN file_group fg on video.file_group_id = fg.id
- WHERE
- published_datetime >= date_trunc('month', months.a)
- AND published_datetime < date_trunc('month', months.a) + interval '1 month'
- AND published_datetime IS NOT NULL
- GROUP BY
- 1
- ORDER BY
- 1
- ''')
+ SELECT DATE_TRUNC('month', months.a),
+ COUNT(video.id)::BIGINT,
+ SUM(size)::BIGINT AS "size"
+ FROM generate_series(
+ date_trunc('month', current_date) - interval '2 years',
+ date_trunc('month', current_date) - interval '1 month',
+ '1 month'::interval) AS months(a),
+ video
+ LEFT JOIN file_group fg on video.file_group_id = fg.id
+ WHERE published_datetime >= date_trunc('month', months.a)
+ AND published_datetime < date_trunc('month', months.a) + interval '1 month'
+ AND published_datetime IS NOT NULL
+ GROUP BY 1
+ ORDER BY 1
+ ''')
monthly_videos = [dict(i) for i in curs.fetchall()]
historical_stats = dict(monthly_videos=monthly_videos)
@@ -651,13 +676,11 @@ async def get_statistics():
if monthly_videos else 0
curs.execute('''
- SELECT
- COUNT(c.id) AS "channels",
- COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels"
- FROM
- channel c
- LEFT JOIN public.tag t on t.id = c.tag_id
- ''')
+ SELECT COUNT(c.id) AS "channels",
+ COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels"
+ FROM channel c
+ LEFT JOIN public.tag t on t.id = c.tag_id
+ ''')
channel_stats = dict(curs.fetchone())
ret = dict(statistics=dict(
videos=video_stats,
@@ -766,3 +789,41 @@ def format_videos_destination(channel_name: str = None, channel_tag: str = None,
videos_destination = get_media_directory() / videos_destination.lstrip('/')
return videos_destination
+
+
+def get_browser_profiles(home: pathlib.Path = WROLPI_HOME) -> dict:
+ """Searches the provided home directory for Chromium and Firefox profiles."""
+ logger.debug(f'Searching for browser profiles in {home}')
+
+ profiles = dict(chromium_profiles=[], firefox_profiles=[])
+
+ chromium_directory = home / '.config/chromium'
+ default_chromium_profile = chromium_directory / 'Default'
+
+ if default_chromium_profile.is_dir():
+ profiles['chromium_profiles'].append(default_chromium_profile)
+ chromium_profiles = chromium_directory.glob('Profile *')
+ for chromium_profile in chromium_profiles:
+ if chromium_profile.is_dir():
+ profiles['chromium_profiles'].append(chromium_profile)
+
+ firefox_profiles_directory = home / '.mozilla/firefox'
+ firefox_profiles_file = firefox_profiles_directory / 'profiles.ini'
+ if firefox_profiles_file.is_file():
+ # Read the profile.ini and get the default profile path.
+ firefox_profiles_file = firefox_profiles_file.read_text()
+
+ default_firefox_profile = re.findall(r'Default=(.*)', firefox_profiles_file)
+ if default_firefox_profile:
+ default_firefox_profile = firefox_profiles_directory / default_firefox_profile[0]
+
+ if default_firefox_profile.is_dir():
+ profiles['firefox_profiles'].append(default_firefox_profile)
+
+ return profiles
+
+
+def browser_profile_to_yt_dlp_arg(profile: pathlib.Path) -> str:
+ """Takes a Path and returns a string that can be used as a yt-dlp `--cookies-from-browser` argument."""
+ *_, browser, profile = str(profile).split('/')
+ return f'{browser}:{profile}'
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index 5fe193a3..2b152663 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -3,12 +3,14 @@ import json
import shutil
from copy import copy
from http import HTTPStatus
+from pathlib import Path
from unittest import mock
import pytest
from modules.videos.downloader import VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader, preview_filename
+from modules.videos.lib import get_videos_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
from wrolpi.downloader import Download, DownloadResult
@@ -546,3 +548,57 @@ def test_normalize_video_file_names(test_directory, video_download_manager):
info_json,
}
assert not vtt.exists()
+
+
+@pytest.mark.asyncio
+async def test_video_download_cookies(test_session, test_directory, mock_video_extract_info, await_switches,
+ video_download_manager, mock_video_process_runner, image_file, test_videos_downloader_config):
+ config = get_videos_downloader_config()
+
+ config.browser_profile = str(test_directory / 'firefox/some directory')
+ (test_directory / 'firefox/some directory').mkdir(parents=True)
+
+ video_path = test_directory / 'a video.mp4'
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
+
+ # Request video download with cookies.
+ settings = {'use_browser_profile': True}
+
+ url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
+ with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
+ mock_video_extract_info.return_value = example_video_json
+ mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+
+ video_download_manager.create_download(url, video_downloader.name, settings=settings)
+ await video_download_manager.wait_for_all_downloads()
+
+ download, cmd, out_dir = mock_video_process_runner.call_args[0]
+
+ assert '--cookies-from-browser' in cmd
+ assert 'firefox:some directory' in cmd
+
+
+@pytest.mark.asyncio
+async def test_video_download_always_use_cookies(test_session, test_directory, mock_video_extract_info, await_switches,
+ video_download_manager, mock_video_process_runner, image_file, test_videos_downloader_config):
+ config = get_videos_downloader_config()
+
+ config.always_use_browser_profile = True
+ config.browser_profile = str(test_directory / 'chromium/some directory')
+ (test_directory / 'chromium/some directory').mkdir(parents=True)
+
+ video_path = test_directory / 'a video.mp4'
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
+
+ url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
+ with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
+ mock_video_extract_info.return_value = example_video_json
+ mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+
+ video_download_manager.create_download(url, video_downloader.name)
+ await video_download_manager.wait_for_all_downloads()
+
+ download, cmd, out_dir = mock_video_process_runner.call_args[0]
+
+ assert '--cookies-from-browser' in cmd
+ assert 'chromium:some directory' in cmd
diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py
index 5fec7ce7..182803a5 100644
--- a/modules/videos/test/test_lib.py
+++ b/modules/videos/test/test_lib.py
@@ -303,3 +303,15 @@ async def test_videos_downloader_config_api(async_client, test_directory, test_v
request, response = await async_client.post('/api/config?file_name=videos_downloader.yaml', json=body)
assert response.status == HTTPStatus.BAD_REQUEST
assert 'Invalid config' in response.json.get('error')
+
+
+@pytest.mark.parametrize('file_name,expected', [
+ ('/home/wrolpi/.config/chromium/Default', 'chromium:Default'),
+ ('/home/wrolpi/.mozilla/firefox/29el0wk0.default-release', 'firefox:29el0wk0.default-release'),
+])
+def test_browser_profile_to_yt_dlp_arg(file_name, expected):
+ """
+ Test the conversion of a browser profile to yt-dlp arguments.
+ """
+ file_name = pathlib.Path(file_name)
+ assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected
diff --git a/requirements.txt b/requirements.txt
index 97b28054..fbcf3f41 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,4 +29,4 @@ tqdm==4.67.1
vininfo[cli]==1.8.0
websockets==13.1
webvtt-py==0.5.1
-yt-dlp==2025.03.27
\ No newline at end of file
+yt-dlp==2025.04.30
diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py
index 09f4a5d5..15f21623 100644
--- a/wrolpi/cmd.py
+++ b/wrolpi/cmd.py
@@ -3,6 +3,7 @@ import asyncio
import dataclasses
import os
import pathlib
+import shlex
import shutil
import tempfile
from asyncio import CancelledError
@@ -111,10 +112,10 @@ TESTING_RUN_COMMAND_RESULT = None
async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None,
- timeout: int = 600, debug: bool = False) -> CommandResult:
+ timeout: int = 600, log_command: bool = True) -> CommandResult:
"""Run a shell command, return the results (stdout/stderr/return code).
- :param debug: Enable debug logging (disabled by default because this is used by very frequent commands).
+ :param log_command: Enable debug logging.
"""
if not isinstance(cmd, (list, tuple)):
raise RuntimeError('Command must be a list or tuple')
@@ -126,10 +127,11 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
with tempfile.NamedTemporaryFile() as stdout_fh, tempfile.NamedTemporaryFile() as stderr_fh:
stdout_file, stderr_file = pathlib.Path(stdout_fh.name), pathlib.Path(stderr_fh.name)
cmd = tuple(str(i) for i in cmd)
- if debug:
- logger.debug(f'run_command: running ({timeout=}): {cmd=}')
+ cmd_str = ' '.join(shlex.quote(i) for i in cmd)
+ if log_command:
+ logger.info(f'run_command: running ({timeout=}): {cmd_str}')
elif logger.isEnabledFor(TRACE_LEVEL):
- logger.trace(f'run_command: running ({timeout=}): {cmd=}')
+ logger.trace(f'run_command: running ({timeout=}): {cmd_str}')
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=stdout_fh, # Use stdout/stderr files to avoid buffer filling up.
@@ -166,10 +168,10 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
stdout = stdout_file.read_bytes() or b''
stderr = stderr_file.read_bytes() or b''
# Logs details of the call, but only if it took a long time or TRACE is enabled.
- if debug:
- logger.debug(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd=}')
+ if log_command:
+ logger.debug(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
elif logger.isEnabledFor(TRACE_LEVEL):
- logger.trace(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd=}')
+ logger.trace(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
return CommandResult(
return_code=proc.returncode,
cancelled=cancelled,
diff --git a/wrolpi/common.py b/wrolpi/common.py
index d5601498..c682c4a4 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -663,13 +663,13 @@ class ConfigFile:
logger.debug(f'Failed to validate config', exc_info=e)
return False
- def is_valid(self, file: pathlib.Path = None) -> bool:
+ def is_valid(self, file: pathlib.Path = None) -> bool | None:
if not self.validator:
raise NotImplementedError(f'Cannot validate {self.file_name} without validator!')
file = file or self.get_file()
- if not file.is_file():
- return False
+ if not file.is_file() or file.stat().st_size == 0:
+ return None
try:
config_data = self.read_config_file(file)
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index ce66790b..6d2fb384 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -30,7 +30,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which, run_command, CommandResult
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
- get_download_info, trim_file_name, get_wrolpi_config
+ get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError, DownloadError
@@ -299,7 +299,7 @@ class Downloader:
raise NotImplementedError()
async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
- timeout: int = None) -> CommandResult:
+ timeout: int = None, debug: bool = True) -> CommandResult:
"""
Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
@@ -309,8 +309,16 @@ class Downloader:
raise RuntimeError('cwd directory does not exist')
timeout = get_wrolpi_config().download_timeout or timeout or self.timeout
- coro = run_command(cmd, cwd=cwd, timeout=timeout)
- return await self.cancel_wrapper(coro, download)
+ coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug)
+ result = await self.cancel_wrapper(coro, download)
+
+ if logger.isEnabledFor(TRACE_LEVEL):
+ for line in result.stdout.decode().splitlines():
+ logger.trace(line)
+ for line in result.stderr.decode().splitlines():
+ logger.error(line)
+
+ return result
@staticmethod
async def cancel_wrapper(coro: Coroutine, download: Download):
@@ -978,14 +986,12 @@ class DownloadManager:
once_downloads = list(map(dict, curs.fetchall()))
stmt = '''
- SELECT
- COUNT(id) FILTER (
- WHERE frequency IS NULL
- AND (status = 'pending' OR status = 'new')
- )
- FROM
- download
- '''
+ SELECT COUNT(id) FILTER (
+ WHERE frequency IS NULL
+ AND (status = 'pending' OR status = 'new')
+ )
+ FROM download \
+ '''
curs.execute(stmt)
pending_once_downloads = curs.fetchone()[0]
@@ -1002,11 +1008,10 @@ class DownloadManager:
"""
with get_db_curs() as curs:
stmt = """
- SELECT
- COUNT(*) FILTER (WHERE status = 'pending') AS pending_downloads,
- COUNT(*) FILTER (WHERE frequency IS NOT NULL) AS recurring_downloads
- FROM download
- """
+ SELECT COUNT(*) FILTER (WHERE status = 'pending') AS pending_downloads,
+ COUNT(*) FILTER (WHERE frequency IS NOT NULL) AS recurring_downloads
+ FROM download \
+ """
curs.execute(stmt)
counts = list(map(dict, curs.fetchall()))[0]
summary = dict(
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index edfe044c..c60574ad 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -222,6 +222,7 @@ class DownloadSettings:
suffix: Optional[str] = None
title_exclude: Optional[str] = None
title_include: Optional[str] = None
+ use_browser_profile: Optional[bool] = False
video_count_limit: Optional[int] = None
video_format: Optional[str] = None
video_resolutions: List[str] = field(default_factory=list)
@@ -239,6 +240,9 @@ class DownloadSettings:
if not self.download_metadata_only:
del self.download_metadata_only
+ if not self.use_browser_profile:
+ del self.use_browser_profile
+
if self.download_order not in (None, 'newest', 'oldest', 'views'):
raise ValidationError(f'Download order must be one of newest, oldest, views, or null.')
diff --git a/wrolpi/status.py b/wrolpi/status.py
index 86ddbb82..508f166e 100755
--- a/wrolpi/status.py
+++ b/wrolpi/status.py
@@ -92,6 +92,7 @@ async def get_iostat_stats() -> IostatInfo:
try:
result = await run_command(
('iostat', '3', '2'), # 3 second interval to gather more data, get 2 count.
+ log_command=False,
)
if result.stdout and result.return_code == 0:
iostat_stats = jc.parse('iostat', result.stdout.decode(), quiet=True)
@@ -222,7 +223,7 @@ IGNORED_PROCESS_COMMANDS = {' '.join(PS_CMD), ''}
async def get_processes_stats() -> List[ProcessInfo]:
- result = await run_command(PS_CMD)
+ result = await run_command(PS_CMD, log_command=False)
# Do not include `ps` in the top processes.
ps_pid = result.pid
@@ -550,7 +551,7 @@ async def get_power_stats() -> PowerStats:
try:
cmd = ('dmesg',)
- result = await run_command(cmd)
+ result = await run_command(cmd, log_command=False)
if result.stdout and result.return_code == 0:
under_voltage = b' Undervoltage detected' in result.stdout
over_current = b' over-current change ' in result.stdout
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index c9159200..100ca4ca 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -924,7 +924,7 @@ async def test_config_lifecycle(await_switches, test_wrolpi_config):
# Config is not imported, default values are used.
assert config.successful_import is False
assert config.wrol_mode is False
- assert config.is_valid() is False
+ assert config.is_valid() is None
assert config.version == 0
assert config.zims_destination
@@ -998,7 +998,7 @@ async def test_config_lifecycle(await_switches, test_wrolpi_config):
async def test_config_valid(async_client, test_wrolpi_config):
# Config file does not yet exist.
config = get_wrolpi_config()
- assert config.is_valid() is False
+ assert config.is_valid() is None
config.dump_config()
assert config.is_valid() is True
diff --git a/wrolpi/test/test_config_api.py b/wrolpi/test/test_config_api.py
index e8d363a3..6c00b398 100644
--- a/wrolpi/test/test_config_api.py
+++ b/wrolpi/test/test_config_api.py
@@ -24,7 +24,7 @@ async def test_configs_crud(async_client, test_session, test_tags_config, tag_fa
assert response.status_code == HTTPStatus.OK
# WROLPiConfig was not saved.
assert 'wrolpi.yaml' in response.json['configs']
- assert response.json['configs']['wrolpi.yaml']['valid'] is False
+ assert response.json['configs']['wrolpi.yaml']['valid'] is None
assert response.json['configs']['wrolpi.yaml']['successful_import'] is False
# Tags was saved, and is valid.
assert 'tags.yaml' in response.json['configs']
diff --git a/wrolpi/vars.py b/wrolpi/vars.py
index 3eaf2285..a9b8e95e 100644
--- a/wrolpi/vars.py
+++ b/wrolpi/vars.py
@@ -115,3 +115,5 @@ IS_MACOS = os.uname()[0] == 'Darwin'
SIMULTANEOUS_DOWNLOAD_DOMAINS = int(os.environ.get('SIMULTANEOUS_DOWNLOAD_DOMAINS',
2 if IS_RPI else 4))
+
+WROLPI_HOME = pathlib.Path(os.environ.get('WROLPI_HOME') or '/home/wrolpi')