Searching wrolpi user's home directory for browser profiles.

app/src/api.js

@@ -419,6 +419,37 @@ export async function updateVideoDownloaderConfig(config) {
     return await updateConfig('videos_downloader.yaml', config);
 }
 
+export async function fetchBrowserProfiles() {
+    const response = await apiGet(`${API_URI}/videos/browser-profiles`);
+    if (response.status === 200) {
+        return await response.json();
+    } else {
+        const message = await getErrorMessage(response, 'Failed to get browser profiles.');
+        toast({
+            type: 'error',
+            title: 'Fetching Browser Profiles Failed',
+            description: message,
+            time: 5000,
+        });
+        return {}
+    }
+}
+
+export async function updateVideoDownloaderSettings(settings) {
+    const response = await apiPut(`${API_URI}/videos/settings`, settings);
+    if (response.status === 200) {
+        return await response.json();
+    } else {
+        const message = await getErrorMessage(response, 'Failed to save settings.');
+        toast({
+            type: 'error',
+            title: 'Saving Settings Failed',
+            description: message,
+            time: 5000,
+        });
+    }
+}
+
 export async function validateRegex(regex) {
     const response = await apiPost(`${API_URI}/valid_regex`, {regex});
     try {

app/src/components/Download.js

@@ -1,14 +1,6 @@
 import React, {useState} from "react";
-import {getDownloaders, postDownload, putDownload} from "../api";
-import {
-    APIButton,
-    DirectorySearch,
-    InfoHeader,
-    InfoPopup,
-    mergeDeep,
-    RequiredAsterisk,
-    useLocalStorage
-} from "./Common";
+import {fetchVideoDownloaderConfig, getDownloaders, postDownload, putDownload} from "../api";
+import {APIButton, DirectorySearch, InfoHeader, InfoPopup, mergeDeep, RequiredAsterisk, useLocalStorage} from "./Common";
 import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
 import Message from "semantic-ui-react/dist/commonjs/collections/Message";
 import {Button, Form, FormInput, Header} from "./Theme";
@@ -16,7 +8,7 @@ import {Form as SForm, FormDropdown} from "semantic-ui-react";
 import {Link} from "react-router-dom";
 import {TagsSelector} from "../Tags";
 import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
-import {commaSeparatedValidator, InputForm, NumberInputForm, UrlInput, UrlsTextarea, useForm} from "../hooks/useForm";
+import {commaSeparatedValidator, InputForm, NumberInputForm, ToggleForm, UrlInput, UrlsTextarea, useForm} from "../hooks/useForm";
 import {
     channelFrequencyOptions,
     days30Option,
@@ -263,7 +255,7 @@ function VideoDownloadCountLimit({form, name = 'video_count_limit', path = 'sett
 }
 
 export function VideoResolutionSelectorForm({form, name = 'video_resolutions', path = 'settings.video_resolutions'}) {
-    const [inputProps, inputAttrs] = form.getSelectionProps({name, path});
+    const [inputProps, inputAttrs] = form.getSelectionProps({name, path, type: 'array'});
 
     return <>
         <InfoHeader
@@ -341,6 +333,34 @@ export function ChannelTagNameForm({form}) {
     </>
 }
 
+export function UseBrowserProfile({form}) {
+    const [config, setConfig] = useState(null);
+
+    const localFetchConfig = async () => {
+        const result = await fetchVideoDownloaderConfig();
+        setConfig(result);
+    }
+
+    React.useEffect(() => {
+        localFetchConfig();
+    }, []);
+
+    const popupContent = <>Use the browser profile to download videos.
+        This is useful for downloading videos that require a login. See: <i>Videos > Settings</i></>;
+    const label = <InfoHeader
+        headerSize='h4'
+        headerContent='Use Browser Profile'
+        popupContent={popupContent}
+    />;
+    return <ToggleForm
+        form={form}
+        label={label}
+        disabled={!config?.browser_profile}
+        name='use_browser_profile'
+        path='settings.use_browser_profile'
+    />
+}
+
 export function VideosDownloadForm({singleDownload = true, onCancel}) {
     const [showMessage, setShowMessage] = React.useState(false);
 
@@ -354,8 +374,9 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
         tag_names: [],
         downloader: Downloaders.Video,
         settings: {
-            video_resolutions: defaultVideoResolutions,
+            use_browser_profile: false,
             video_format: defaultVideoFormat,
+            video_resolutions: defaultVideoResolutions,
         }
     }
 
@@ -437,6 +458,11 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
                     <VideoFormatSelectorForm form={form}/>
                 </Grid.Column>
             </Grid.Row>
+            <Grid.Row>
+                <Grid.Column>
+                    <UseBrowserProfile form={form}/>
+                </Grid.Column>
+            </Grid.Row>
             <Grid.Row columns={1}>
                 <Grid.Column>
                     <ChannelTagNameForm form={form}/>
@@ -527,9 +553,10 @@ export function ChannelDownloadForm({
             minimum_duration: null,
             title_exclude: null,
             title_include: null,
+            use_browser_profile: false,
             video_count_limit: null,
-            video_resolutions: defaultVideoResolutions,
             video_format: defaultVideoFormat,
+            video_resolutions: defaultVideoResolutions,
         },
         sub_downloader: Downloaders.Video,
         tag_names: [],
@@ -622,6 +649,11 @@ export function ChannelDownloadForm({
                     <VideoMaximumDurationForm form={form}/>
                 </Grid.Column>
             </Grid.Row>
+            <Grid.Row>
+                <Grid.Column>
+                    <UseBrowserProfile form={form}/>
+                </Grid.Column>
+            </Grid.Row>
             <Grid.Row columns={1}>
                 <Grid.Column>
                     <ChannelTagNameForm form={form}/>

app/src/components/Videos.js

@@ -11,6 +11,7 @@ import {
     findPosterPath,
     humanFileSize,
     humanNumber,
+    InfoHeader,
     isoDatetimeToAgoPopup,
     mimetypeColor,
     PageContainer,
@@ -30,6 +31,7 @@ import {
     CardHeader,
     Container,
     Form,
+    FormDropdown,
     Image,
     PlaceholderHeader,
     PlaceholderLine,
@@ -43,7 +45,7 @@ import {FileRowTagIcon, FilesView} from "./Files";
 import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
 import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
 import {Button, Card, Header, Loader, Placeholder, Popup, Segment, Statistic, StatisticGroup} from "./Theme";
-import {deleteVideos, fetchVideoDownloaderConfig, postVideoFileFormat, updateVideoDownloaderConfig} from "../api";
+import {deleteVideos, fetchBrowserProfiles, fetchVideoDownloaderConfig, postVideoFileFormat, updateVideoDownloaderConfig} from "../api";
 import {Media, QueryContext, ThemeContext} from "../contexts/contexts";
 import _ from "lodash";
 import {defaultFileOrder, defaultSearchOrder} from "./Vars";
@@ -257,43 +259,76 @@ function VideosSettings() {
             writesubtitles: true,
             writethumbnail: true,
         },
+        always_use_browser_profile: false,
+        yt_dlp_extra_args: '',
+        browser_profile: '',
     };
 
-    const fetcher = async () => {
-        return await fetchVideoDownloaderConfig();
-    }
+    const configSubmitter = async () => {
+        return await updateVideoDownloaderConfig(configForm.formData);
+    };
 
-    const submitter = async () => {
-        return await updateVideoDownloaderConfig(form.formData);
-    }
-
-    const form = useForm({
-        fetcher,
-        submitter,
+    const configForm = useForm({
+        fetcher: fetchVideoDownloaderConfig,
+        submitter: configSubmitter,
         emptyFormData,
     });
 
+    const emptyProfilesOptions = [{key: null, text: 'No profiles found'}];
+    const loadingProfilesOptions = [{key: null, text: 'Loading profiles, please wait...'}];
+    const [browserProfilesOptions, setBrowserProfilesOptions] = useState(loadingProfilesOptions);
+
+    const localFetchBrowserProfiles = async () => {
+        let tempProfiles = [];
+        let localBrowserProfiles = await fetchBrowserProfiles();
+
+        const chromiumProfiles = localBrowserProfiles['chromium_profiles'];
+        for (const value of Object.values(chromiumProfiles)) {
+            tempProfiles.push({value: value, text: value});
+        }
+        const firefoxProfiles = localBrowserProfiles['firefox_profiles'];
+        for (const value of Object.values(firefoxProfiles)) {
+            tempProfiles.push({value: value, text: value});
+        }
+
+        if (_.isEmpty(tempProfiles)) {
+            tempProfiles = emptyProfilesOptions;
+        }
+        console.debug('VideosSettings: Got browser profiles:', tempProfiles);
+        setBrowserProfilesOptions(tempProfiles);
+    };
+
+    React.useEffect(() => {
+        localFetchBrowserProfiles();
+    }, []);
+
+    const alwaysUseBrowserProfileLabel = <InfoHeader
+        headerSize='h5'
+        headerContent='Always Use Browser Profile'
+        popupContent='Always download videos with the selected browser profile. This is risky, and therefore discouraged.'
+    />;
+
     return <Segment>
-        <Header as='h3'>Video Downloader Settings</Header>
+        <Header as='h3'>Video Downloader Config</Header>
 
         <Form>
             <Grid>
                 <Grid.Row columns={2}>
                     <Grid.Column mobile={16} computer={8}>
                         <VideoResolutionSelectorForm
-                            form={form}
+                            form={configForm}
                             name='video_resolutions'
                             path='video_resolutions'
                         />
                     </Grid.Column>
                     <Grid.Column mobile={16} computer={8}>
-                        <VideoFileNameForm form={form}/>
+                        <VideoFileNameForm form={configForm}/>
                     </Grid.Column>
                 </Grid.Row>
                 <Grid.Row columns={2}>
                     <Grid.Column mobile={16} computer={8}>
                         <ToggleForm
-                            form={form}
+                            form={configForm}
                             label='Do not overwrite existing files'
                             name='nooverwrites'
                             path='yt_dlp_options.nooverwrites'
@@ -302,7 +337,7 @@ function VideosSettings() {
                     </Grid.Column>
                     <Grid.Column mobile={16} computer={8}>
                         <ToggleForm
-                            form={form}
+                            form={configForm}
                             label='Download automatic subtitles'
                             name='writeautomaticsub'
                             path='yt_dlp_options.writeautomaticsub'
@@ -313,7 +348,7 @@ function VideosSettings() {
                 <Grid.Row>
                     <Grid.Column mobile={16} computer={8}>
                         <ToggleForm
-                            form={form}
+                            form={configForm}
                             label='Download subtitles'
                             name='writesubtitles'
                             path='yt_dlp_options.writesubtitles'
@@ -322,7 +357,7 @@ function VideosSettings() {
                     </Grid.Column>
                     <Grid.Column mobile={16} computer={8}>
                         <ToggleForm
-                            form={form}
+                            form={configForm}
                             label='Download thumbnail'
                             name='writethumbnail'
                             path='yt_dlp_options.writethumbnail'
@@ -330,13 +365,41 @@ function VideosSettings() {
                         />
                     </Grid.Column>
                 </Grid.Row>
+                <Grid.Row>
+                    <Grid.Column mobile={16} computer={8}>
+                        <InputForm
+                            form={configForm}
+                            name='yt_dlp_extra_args'
+                            path='yt_dlp_extra_args'
+                            label='Extra yt-dlp Arguments'
+                            placeholder='--cookies-from-browser chromium'
+                            icon='terminal'
+                        />
+                    </Grid.Column>
+                    <Grid.Column mobile={16} computer={8}>
+                        <VideoBrowserCookiesSelector form={configForm} options={browserProfilesOptions}/>
+                    </Grid.Column>
+                </Grid.Row>
+                <Grid.Row>
+                    <Grid.Column mobile={16} computer={8}/>
+                    <Grid.Column mobile={16} computer={8}>
+                        <ToggleForm
+                            form={configForm}
+                            label={alwaysUseBrowserProfileLabel}
+                            name='always_use_browser_profile'
+                            path='always_use_browser_profile'
+                            disabled={!configForm.formData.browser_profile}
+                        />
+                    </Grid.Column>
+                </Grid.Row>
+
                 <Grid.Row columns={1}>
                     <Grid.Column textAlign='right'>
                         <APIButton
-                            disabled={form.disabled || !form.ready}
+                            disabled={configForm.disabled || !configForm.ready}
                             type='submit'
                             style={{marginTop: '0.5em'}}
-                            onClick={form.onSubmit}
+                            onClick={configForm.onSubmit}
                             id='video_settings_save_button'
                         >Save</APIButton>
                     </Grid.Column>
@@ -346,6 +409,39 @@ function VideosSettings() {
     </Segment>
 }
 
+
+export function VideoBrowserCookiesSelector({
+                                                form,
+                                                options,
+                                                name = 'browser_profile',
+                                                path = 'browser_profile',
+                                            }) {
+    const [inputProps, inputAttrs] = form.getSelectionProps({name, path});
+
+    const popupContent = <div>
+        Select a browser profile to use for cookies. This is useful for sites that require login to download videos.
+        <br/>
+        <br/>
+        <b>MUST BE FROM THE wrolpi USER!</b>
+    </div>;
+
+    return <>
+        <InfoHeader
+            headerSize='h5'
+            headerContent='Browser Profile'
+            popupContent={popupContent}
+        />
+        <FormDropdown
+            selection
+            placeholder='Select Browser Profile'
+            options={options}
+            name={name}
+            id='browser_profile_dropdown'
+            {...inputProps}
+        />
+    </>
+}
+
 function VideosStatistics() {
     useTitle('Video Statistics');
 

app/src/components/admin/Configs.js

@@ -1,12 +1,18 @@
 import {APIButton} from "../Common";
-import {Icon, Table} from "../Theme";
+import {Icon, Popup, Table} from "../Theme";
 import React from "react";
 import {TableBody, TableCell, TableHeader, TableHeaderCell, TableRow} from "semantic-ui-react";
 import {TableRowPlaceholder} from "../Placeholder";
 import {useNavigate} from "react-router-dom";
 
 function WarningIcon({ok}) {
-    return <Icon color={ok ? 'violet' : 'red'} name={ok ? 'check' : 'close'}/>
+    // null is empty, true is valid, false is invalid.
+    const trigger = <Icon
+        color={ok === true ? 'violet' : ok === false ? 'red' : 'grey'}
+        name={ok === true ? 'check' : ok === false ? 'close' : 'check'}
+    />;
+    const content = ok === true ? 'Valid and can be imported.' : ok === false ? 'Invalid, can be overwritten.' : 'Empty config, can be overwritten.';
+    return <Popup trigger={trigger} content={content}/>
 }
 
 function ConfigTableRow({config, importConfig, saveConfig, fetchConfigs}) {

app/src/hooks/useForm.js

@@ -185,7 +185,7 @@ export function useForm({
         }
         const value = _.get(formData, path); // Get a path, separated by .'s
         if (value === undefined) {
-            patchFormData(_.set(formData, path, null));
+            patchFormData(_.set(formData, path, type === 'array' ? [] : null));
         }
         // Allow bypass of `handleInputEvent` which only handles an input event.
         const setValue = newValue => {
@@ -426,7 +426,7 @@ export function UrlsTextarea({name = 'urls', required, form}) {
     </FormInput>
 }
 
-export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big'}) {
+export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big', disabled=false}) {
     const [inputProps, inputAttrs] = form.getCustomProps({name, path})
 
     const iconElm = icon ? <Icon size={iconSize} name={icon}/> : null;
@@ -434,7 +434,7 @@ export function ToggleForm({form, name, label, path, icon = null, iconSize = 'bi
         label={label}>
         {iconElm}
         <Toggle
-            disabled={form.disabled}
+            disabled={disabled || form.disabled}
             name={name}
             checked={inputProps.value}
             onChange={inputProps.onChange}

help.sh

@@ -324,6 +324,12 @@ else
   echo "FAILED: wget cannot be run"
 fi
 
+if aria2c -h >/dev/null 2>&1; then
+  echo "OK: aria2c can be run"
+else
+  echo "FAILED: aria2c cannot be run"
+fi
+
 if /opt/wrolpi/venv/bin/yt-dlp -h >/dev/null 2>&1; then
   echo "OK: yt-dlp can be run"
 else

modules/archive/lib.py

@@ -565,7 +565,7 @@ async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
                READABILITY_BIN, singlefile_file.name, shlex.quote(url))
         # Docker containers may not have this directory. But, this directory is necessary on RPi.
         cwd = '/home/wrolpi' if os.path.isdir('/home/wrolpi') else None
-        result = await run_command(cmd, cwd=cwd, timeout=timeout, debug=True)
+        result = await run_command(cmd, cwd=cwd, timeout=timeout)
 
         logger.debug(f'readability for {url} exited with {result.return_code}')
         stdout, stderr = result.stdout.decode(), result.stderr.decode()

modules/map/lib.py

@@ -193,7 +193,6 @@ async def run_import_command(*paths: Path):
     cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', *paths)
     result = await run_command(
         cmd,
-        debug=True,
         timeout=0,  # No timeout, map importing could take days.
     )
 

modules/videos/api.py

@@ -66,3 +66,9 @@ async def post_file_format(_: Request, body: schema.VideoFileFormatRequest):
             error=str(e),
             video_file_format=body.video_file_format,
         ), status=HTTPStatus.BAD_REQUEST)
+
+
+@content_bp.get('/browser-profiles')
+async def get_browser_profiles(_: Request):
+    browser_profiles = lib.get_browser_profiles()
+    return json_response(browser_profiles)

modules/videos/downloader.py

@@ -32,7 +32,7 @@ from .channel.lib import create_channel, get_channel
 from .common import get_no_channel_directory, update_view_counts_and_censored, \
     ffmpeg_video_complete
 from .errors import UnknownChannel
-from .lib import get_videos_downloader_config, YDL, ydl_logger, format_videos_destination
+from .lib import get_videos_downloader_config, YDL, ydl_logger, format_videos_destination, browser_profile_to_yt_dlp_arg
 from .models import Video, Channel
 from .normalize_video_url import normalize_video_url
 from .schema import ChannelPostRequest
@@ -45,7 +45,7 @@ VIDEO_RESOLUTION_MAP = {
     '480p': ['135+bestaudio', '231+bestaudio', '135+139', 'mp4-480p', 'res:480'],
     '720p': [
         '298+bestaudio', '232+bestaudio',  # 720@60 avc1 + best audio
-        '136+bestaudio', '22', 'res:720'],
+        '136+bestaudio', '22', '311+bestaudio', 'res:720'],
     '1080p': [
         '299+bestaudio', '312+bestaudio', '270+bestaudio',  # 1080@60 avc1 + best audio
         '137+bestaudio', '614+bestaudio', 'res:1080'],
@@ -328,7 +328,6 @@ class VideoDownloader(Downloader, ABC):
         if download.attempts >= 10:
             raise UnrecoverableDownloadError('Max download attempts reached')
 
-        download_id = download.id
         url = normalize_video_url(download.url)
 
         # Copy settings into Download (they may be from the ChannelDownloader)
@@ -395,19 +394,13 @@ class VideoDownloader(Downloader, ABC):
                 # User has requested refresh of video metadata, skip the rest of the video downloader.
                 return await self.download_info_json(download, video_path)
 
-            # Do the real download.
             cmd = (
                 str(YT_DLP_BIN),
-                '-c',  # Continue downloads
                 '-f', video_resolutions,
                 '--match-filter', '!is_live',  # Do not attempt to download Live videos.
-                '--write-subs',
-                '--write-auto-subs',
                 '--sub-format', DEFAULT_CAPTION_FORMAT,
                 '--convert-subs', DEFAULT_CAPTION_FORMAT,
                 '--convert-thumbnails', DEFAULT_POSTER_FORMAT,
-                '--write-thumbnail',
-                '--write-info-json',
                 '--merge-output-format', video_format,
                 # '--remux-video', video_format,
                 '--no-cache-dir',
@@ -417,10 +410,32 @@ class VideoDownloader(Downloader, ABC):
                 '--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
                 # Use experimental feature to merge files.
                 '--ppa', 'Merger+ffmpeg_o1:-strict -2',
-                '-o', video_path,
-                url,
             )
-            result = await self.process_runner(download, cmd, out_dir)
+            if config.continue_dl:
+                cmd = (*cmd, '--continue')
+            if config.writesubtitles:
+                cmd = (*cmd, '--write-subs', '--write-auto-subs')
+            if config.writethumbnail:
+                cmd = (*cmd, '--write-thumbnail')
+            if config.writeinfojson:
+                cmd = (*cmd, '--write-info-json')
+            if config.yt_dlp_extra_args:
+                cmd = (*cmd, *config.yt_dlp_extra_args.split(' '))
+            if config.browser_profile and (config.always_use_browser_profile or settings.get('use_browser_profile')):
+                browser_profile = pathlib.Path(config.browser_profile)
+                if browser_profile.exists():
+                    browser_profile = browser_profile_to_yt_dlp_arg(browser_profile)
+                    cmd = (*cmd, '--cookies-from-browser', browser_profile)
+                else:
+                    logger.error(f'Browser profile {config.browser_profile} does not exist')
+
+            # Add destination and the video's URL to the command.
+            cmd = (*cmd,
+                   '-o', video_path,
+                   url,
+                   )
+            # Do the real download.
+            result = await self.process_runner(download, cmd, out_dir, debug=True)
             stdout = result.stdout.decode()
             stderr = result.stderr.decode()
 

modules/videos/lib.py

@@ -24,7 +24,7 @@ from wrolpi.errors import UnknownDirectory
 from wrolpi.events import Events
 from wrolpi.files.lib import split_path_stem_and_suffix
 from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
-from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
+from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST, WROLPI_HOME
 from .common import is_valid_poster, convert_image, \
     generate_video_poster, ConfigError, \
     extract_video_duration
@@ -380,6 +380,9 @@ class VideoDownloaderConfigValidator:
     video_resolutions: List[str] = field(default_factory=lambda: ['1080p', '720p', '480p', 'maximum'])
     version: int = None
     yt_dlp_options: VideoDownloaderConfigYtDlpOptionsValidator = field(default_factory=dict)
+    yt_dlp_extra_args: str = ''
+    always_use_browser_profile: bool = False
+    browser_profile: str = ''
 
     def __post_init__(self):
         allowed_fields = {i.name for i in dataclasses.fields(VideoDownloaderConfigYtDlpOptionsValidator)}
@@ -405,6 +408,9 @@ class VideoDownloaderConfig(ConfigFile):
             writethumbnail=True,
             youtube_include_dash_manifest=False,
         ),
+        yt_dlp_extra_args='',
+        always_use_browser_profile=False,
+        browser_profile='',
     )
     validator = VideoDownloaderConfigValidator
 
@@ -464,6 +470,30 @@ class VideoDownloaderConfig(ConfigFile):
     def yt_dlp_options(self, value: dict):
         self.update({'yt_dlp_options': value})
 
+    @property
+    def yt_dlp_extra_args(self) -> str:
+        return self._config['yt_dlp_extra_args']
+
+    @yt_dlp_extra_args.setter
+    def yt_dlp_extra_args(self, value: str):
+        self.update({**self._config, 'yt_dlp_extra_args': value})
+
+    @property
+    def browser_profile(self) -> str:
+        return self._config['browser_profile']
+
+    @browser_profile.setter
+    def browser_profile(self, value: str):
+        self.update({**self._config, 'browser_profile': value})
+
+    @property
+    def always_use_browser_profile(self) -> bool:
+        return self._config['always_use_browser_profile']
+
+    @always_use_browser_profile.setter
+    def always_use_browser_profile(self, value: bool):
+        self.update({**self._config, 'always_use_browser_profile': value})
+
     def import_config(self, file: pathlib.Path = None, send_events=False):
         super().import_config(file, send_events)
         self.successful_import = True
@@ -595,53 +625,48 @@ def get_channel_source_id(url: str) -> str:
 async def get_statistics():
     with get_db_curs() as curs:
         curs.execute('''
-        SELECT
-            -- total videos
-            COUNT(v.id) AS "videos",
-            -- total videos downloaded over the past week/month/year
-            COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 week') AS "week",
-            COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 month') AS "month",
-            COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 year') AS "year",
-            -- sum of all video lengths in seconds
-            COALESCE(SUM(fg.length), 0) AS "sum_duration",
-            -- sum of all video file sizes
-            COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size",
-            -- largest video
-            COALESCE(MAX(fg.size), 0) AS "max_size",
-            -- Videos may or may not have comments.
-            COUNT(v.id) FILTER ( WHERE v.have_comments = TRUE ) AS "have_comments",
-            COUNT(v.id) FILTER ( WHERE v.have_comments = FALSE AND v.comments_failed = FALSE
-                and fg.censored = false and fg.url is not null) AS "missing_comments",
-            COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments",
-            COUNT(v.id) FILTER ( WHERE fg.censored = TRUE ) as "censored_videos"
-        FROM
-            video v
-            LEFT JOIN file_group fg on v.file_group_id = fg.id
-        ''')
+                     SELECT
+                         -- total videos
+                         COUNT(v.id)                                                                        AS "videos",
+                         -- total videos downloaded over the past week/month/year
+                         COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 week')  AS "week",
+                         COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 month') AS "month",
+                         COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 year')  AS "year",
+                         -- sum of all video lengths in seconds
+                         COALESCE(SUM(fg.length), 0)                                                        AS "sum_duration",
+                         -- sum of all video file sizes
+                         COALESCE(SUM(fg.size), 0)::BIGINT                                                  AS "sum_size",
+                         -- largest video
+                         COALESCE(MAX(fg.size), 0)                                                          AS "max_size",
+                         -- Videos may or may not have comments.
+                         COUNT(v.id) FILTER ( WHERE v.have_comments = TRUE )                                AS "have_comments",
+                         COUNT(v.id) FILTER ( WHERE v.have_comments = FALSE AND v.comments_failed = FALSE
+                             and fg.censored = false and
+                                                    fg.url is not null)                                     AS "missing_comments",
+                         COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE )                              AS "failed_comments",
+                         COUNT(v.id) FILTER ( WHERE fg.censored = TRUE )                                    as "censored_videos"
+                     FROM video v
+                              LEFT JOIN file_group fg on v.file_group_id = fg.id
+                     ''')
         video_stats = dict(curs.fetchone())
 
         # Get the total videos downloaded every month for the past two years.
         curs.execute('''
-        SELECT
-            DATE_TRUNC('month', months.a),
-            COUNT(video.id)::BIGINT,
-            SUM(size)::BIGINT AS "size"
-        FROM
-            generate_series(
-                date_trunc('month', current_date) - interval '2 years',
-                date_trunc('month', current_date) - interval '1 month',
-                '1 month'::interval) AS months(a),
-            video
-            LEFT JOIN file_group fg on video.file_group_id = fg.id
-        WHERE
-            published_datetime >= date_trunc('month', months.a)
-            AND published_datetime < date_trunc('month', months.a) + interval '1 month'
-            AND published_datetime IS NOT NULL
-        GROUP BY
-            1
-        ORDER BY
-            1
-        ''')
+                     SELECT DATE_TRUNC('month', months.a),
+                            COUNT(video.id)::BIGINT,
+                            SUM(size)::BIGINT AS "size"
+                     FROM generate_series(
+                                  date_trunc('month', current_date) - interval '2 years',
+                                  date_trunc('month', current_date) - interval '1 month',
+                                  '1 month'::interval) AS months(a),
+                          video
+                              LEFT JOIN file_group fg on video.file_group_id = fg.id
+                     WHERE published_datetime >= date_trunc('month', months.a)
+                       AND published_datetime < date_trunc('month', months.a) + interval '1 month'
+                       AND published_datetime IS NOT NULL
+                     GROUP BY 1
+                     ORDER BY 1
+                     ''')
         monthly_videos = [dict(i) for i in curs.fetchall()]
 
         historical_stats = dict(monthly_videos=monthly_videos)
@@ -651,13 +676,11 @@ async def get_statistics():
             if monthly_videos else 0
 
         curs.execute('''
-        SELECT
-            COUNT(c.id) AS "channels",
-            COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels"
-        FROM
-            channel c
-            LEFT JOIN public.tag t on t.id = c.tag_id
-        ''')
+                     SELECT COUNT(c.id)                                       AS "channels",
+                            COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels"
+                     FROM channel c
+                              LEFT JOIN public.tag t on t.id = c.tag_id
+                     ''')
         channel_stats = dict(curs.fetchone())
     ret = dict(statistics=dict(
         videos=video_stats,
@@ -766,3 +789,41 @@ def format_videos_destination(channel_name: str = None, channel_tag: str = None,
     videos_destination = get_media_directory() / videos_destination.lstrip('/')
 
     return videos_destination
+
+
+def get_browser_profiles(home: pathlib.Path = WROLPI_HOME) -> dict:
+    """Searches the provided home directory for Chromium and Firefox profiles."""
+    logger.debug(f'Searching for browser profiles in {home}')
+
+    profiles = dict(chromium_profiles=[], firefox_profiles=[])
+
+    chromium_directory = home / '.config/chromium'
+    default_chromium_profile = chromium_directory / 'Default'
+
+    if default_chromium_profile.is_dir():
+        profiles['chromium_profiles'].append(default_chromium_profile)
+    chromium_profiles = chromium_directory.glob('Profile *')
+    for chromium_profile in chromium_profiles:
+        if chromium_profile.is_dir():
+            profiles['chromium_profiles'].append(chromium_profile)
+
+    firefox_profiles_directory = home / '.mozilla/firefox'
+    firefox_profiles_file = firefox_profiles_directory / 'profiles.ini'
+    if firefox_profiles_file.is_file():
+        # Read the profile.ini and get the default profile path.
+        firefox_profiles_file = firefox_profiles_file.read_text()
+
+        default_firefox_profile = re.findall(r'Default=(.*)', firefox_profiles_file)
+        if default_firefox_profile:
+            default_firefox_profile = firefox_profiles_directory / default_firefox_profile[0]
+
+            if default_firefox_profile.is_dir():
+                profiles['firefox_profiles'].append(default_firefox_profile)
+
+    return profiles
+
+
+def browser_profile_to_yt_dlp_arg(profile: pathlib.Path) -> str:
+    """Takes a Path and returns a string that can be used as a yt-dlp `--cookies-from-browser` argument."""
+    *_, browser, profile = str(profile).split('/')
+    return f'{browser}:{profile}'

modules/videos/test/test_downloader.py

@@ -3,12 +3,14 @@ import json
 import shutil
 from copy import copy
 from http import HTTPStatus
+from pathlib import Path
 from unittest import mock
 
 import pytest
 
 from modules.videos.downloader import VideoDownloader, \
     get_or_create_channel, channel_downloader, video_downloader, preview_filename
+from modules.videos.lib import get_videos_downloader_config
 from modules.videos.models import Channel, Video
 from wrolpi.conftest import test_directory, await_switches
 from wrolpi.downloader import Download, DownloadResult
@@ -546,3 +548,57 @@ def test_normalize_video_file_names(test_directory, video_download_manager):
         info_json,
     }
     assert not vtt.exists()
+
+
+@pytest.mark.asyncio
+async def test_video_download_cookies(test_session, test_directory, mock_video_extract_info, await_switches,
+                                      video_download_manager, mock_video_process_runner, image_file, test_videos_downloader_config):
+    config = get_videos_downloader_config()
+
+    config.browser_profile = str(test_directory / 'firefox/some directory')
+    (test_directory / 'firefox/some directory').mkdir(parents=True)
+
+    video_path = test_directory / 'a video.mp4'
+    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
+
+    # Request video download with cookies.
+    settings = {'use_browser_profile': True}
+
+    url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
+    with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
+        mock_video_extract_info.return_value = example_video_json
+        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+
+        video_download_manager.create_download(url, video_downloader.name, settings=settings)
+        await video_download_manager.wait_for_all_downloads()
+
+        download, cmd, out_dir = mock_video_process_runner.call_args[0]
+
+    assert '--cookies-from-browser' in cmd
+    assert 'firefox:some directory' in cmd
+
+
+@pytest.mark.asyncio
+async def test_video_download_always_use_cookies(test_session, test_directory, mock_video_extract_info, await_switches,
+                                                 video_download_manager, mock_video_process_runner, image_file, test_videos_downloader_config):
+    config = get_videos_downloader_config()
+
+    config.always_use_browser_profile = True
+    config.browser_profile = str(test_directory / 'chromium/some directory')
+    (test_directory / 'chromium/some directory').mkdir(parents=True)
+
+    video_path = test_directory / 'a video.mp4'
+    shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
+
+    url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
+    with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
+        mock_video_extract_info.return_value = example_video_json
+        mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+
+        video_download_manager.create_download(url, video_downloader.name)
+        await video_download_manager.wait_for_all_downloads()
+
+        download, cmd, out_dir = mock_video_process_runner.call_args[0]
+
+    assert '--cookies-from-browser' in cmd
+    assert 'chromium:some directory' in cmd

modules/videos/test/test_lib.py

@@ -303,3 +303,15 @@ async def test_videos_downloader_config_api(async_client, test_directory, test_v
     request, response = await async_client.post('/api/config?file_name=videos_downloader.yaml', json=body)
     assert response.status == HTTPStatus.BAD_REQUEST
     assert 'Invalid config' in response.json.get('error')
+
+
+@pytest.mark.parametrize('file_name,expected', [
+    ('/home/wrolpi/.config/chromium/Default', 'chromium:Default'),
+    ('/home/wrolpi/.mozilla/firefox/29el0wk0.default-release', 'firefox:29el0wk0.default-release'),
+])
+def test_browser_profile_to_yt_dlp_arg(file_name, expected):
+    """
+    Test the conversion of a browser profile to yt-dlp arguments.
+    """
+    file_name = pathlib.Path(file_name)
+    assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected

requirements.txt

@@ -29,4 +29,4 @@ tqdm==4.67.1
 vininfo[cli]==1.8.0
 websockets==13.1
 webvtt-py==0.5.1
-yt-dlp==2025.03.27
+yt-dlp==2025.04.30

wrolpi/cmd.py

@@ -3,6 +3,7 @@ import asyncio
 import dataclasses
 import os
 import pathlib
+import shlex
 import shutil
 import tempfile
 from asyncio import CancelledError
@@ -111,10 +112,10 @@ TESTING_RUN_COMMAND_RESULT = None
 
 
 async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None,
-                      timeout: int = 600, debug: bool = False) -> CommandResult:
+                      timeout: int = 600, log_command: bool = True) -> CommandResult:
     """Run a shell command, return the results (stdout/stderr/return code).
 
-    :param debug: Enable debug logging (disabled by default because this is used by very frequent commands).
+    :param log_command: Enable debug logging.
     """
     if not isinstance(cmd, (list, tuple)):
         raise RuntimeError('Command must be a list or tuple')
@@ -126,10 +127,11 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
     with tempfile.NamedTemporaryFile() as stdout_fh, tempfile.NamedTemporaryFile() as stderr_fh:
         stdout_file, stderr_file = pathlib.Path(stdout_fh.name), pathlib.Path(stderr_fh.name)
         cmd = tuple(str(i) for i in cmd)
-        if debug:
-            logger.debug(f'run_command: running ({timeout=}): {cmd=}')
+        cmd_str = ' '.join(shlex.quote(i) for i in cmd)
+        if log_command:
+            logger.info(f'run_command: running ({timeout=}): {cmd_str}')
         elif logger.isEnabledFor(TRACE_LEVEL):
-            logger.trace(f'run_command: running ({timeout=}): {cmd=}')
+            logger.trace(f'run_command: running ({timeout=}): {cmd_str}')
         proc = await asyncio.create_subprocess_exec(
             *cmd,
             stdout=stdout_fh,  # Use stdout/stderr files to avoid buffer filling up.
@@ -166,10 +168,10 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
         stdout = stdout_file.read_bytes() or b''
         stderr = stderr_file.read_bytes() or b''
         # Logs details of the call, but only if it took a long time or TRACE is enabled.
-        if debug:
-            logger.debug(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd=}')
+        if log_command:
+            logger.debug(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
         elif logger.isEnabledFor(TRACE_LEVEL):
-            logger.trace(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd=}')
+            logger.trace(f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
         return CommandResult(
             return_code=proc.returncode,
             cancelled=cancelled,

wrolpi/common.py

@@ -663,13 +663,13 @@ class ConfigFile:
             logger.debug(f'Failed to validate config', exc_info=e)
             return False
 
-    def is_valid(self, file: pathlib.Path = None) -> bool:
+    def is_valid(self, file: pathlib.Path = None) -> bool | None:
         if not self.validator:
             raise NotImplementedError(f'Cannot validate {self.file_name} without validator!')
 
         file = file or self.get_file()
-        if not file.is_file():
-            return False
+        if not file.is_file() or file.stat().st_size == 0:
+            return None
 
         try:
             config_data = self.read_config_file(file)

wrolpi/downloader.py

@@ -30,7 +30,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
 from wrolpi.cmd import which, run_command, CommandResult
 from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
     wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
-    get_download_info, trim_file_name, get_wrolpi_config
+    get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL
 from wrolpi.dates import TZDateTime, now, Seconds
 from wrolpi.db import get_db_session, get_db_curs, optional_session
 from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError, DownloadError
@@ -299,7 +299,7 @@ class Downloader:
         raise NotImplementedError()
 
     async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
-                             timeout: int = None) -> CommandResult:
+                             timeout: int = None, debug: bool = True) -> CommandResult:
         """
         Run a subprocess using the provided arguments.  This process can be killed by the Download Manager.
 
@@ -309,8 +309,16 @@ class Downloader:
             raise RuntimeError('cwd directory does not exist')
 
         timeout = get_wrolpi_config().download_timeout or timeout or self.timeout
-        coro = run_command(cmd, cwd=cwd, timeout=timeout)
-        return await self.cancel_wrapper(coro, download)
+        coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug)
+        result = await self.cancel_wrapper(coro, download)
+
+        if logger.isEnabledFor(TRACE_LEVEL):
+            for line in result.stdout.decode().splitlines():
+                logger.trace(line)
+            for line in result.stderr.decode().splitlines():
+                logger.error(line)
+
+        return result
 
     @staticmethod
     async def cancel_wrapper(coro: Coroutine, download: Download):
@@ -978,14 +986,12 @@ class DownloadManager:
             once_downloads = list(map(dict, curs.fetchall()))
 
             stmt = '''
-                SELECT
-                    COUNT(id) FILTER (
-                        WHERE frequency IS NULL
-                        AND (status = 'pending' OR status = 'new')
-                        )
-                FROM
-                    download
-            '''
+                   SELECT COUNT(id) FILTER (
+                       WHERE frequency IS NULL
+                           AND (status = 'pending' OR status = 'new')
+                       )
+                   FROM download \
+                   '''
             curs.execute(stmt)
             pending_once_downloads = curs.fetchone()[0]
 
@@ -1002,11 +1008,10 @@ class DownloadManager:
         """
         with get_db_curs() as curs:
             stmt = """
-                SELECT
-                 COUNT(*) FILTER (WHERE status = 'pending') AS pending_downloads,
-                 COUNT(*) FILTER (WHERE frequency IS NOT NULL) AS recurring_downloads
-                FROM download
-                """
+                   SELECT COUNT(*) FILTER (WHERE status = 'pending')    AS pending_downloads,
+                          COUNT(*) FILTER (WHERE frequency IS NOT NULL) AS recurring_downloads
+                   FROM download \
+                   """
             curs.execute(stmt)
             counts = list(map(dict, curs.fetchall()))[0]
         summary = dict(

wrolpi/schema.py

@@ -222,6 +222,7 @@ class DownloadSettings:
     suffix: Optional[str] = None
     title_exclude: Optional[str] = None
     title_include: Optional[str] = None
+    use_browser_profile: Optional[bool] = False
     video_count_limit: Optional[int] = None
     video_format: Optional[str] = None
     video_resolutions: List[str] = field(default_factory=list)
@@ -239,6 +240,9 @@ class DownloadSettings:
         if not self.download_metadata_only:
             del self.download_metadata_only
 
+        if not self.use_browser_profile:
+            del self.use_browser_profile
+
         if self.download_order not in (None, 'newest', 'oldest', 'views'):
             raise ValidationError(f'Download order must be one of newest, oldest, views, or null.')
 

wrolpi/status.py

@@ -92,6 +92,7 @@ async def get_iostat_stats() -> IostatInfo:
     try:
         result = await run_command(
             ('iostat', '3', '2'),  # 3 second interval to gather more data, get 2 count.
+            log_command=False,
         )
         if result.stdout and result.return_code == 0:
             iostat_stats = jc.parse('iostat', result.stdout.decode(), quiet=True)
@@ -222,7 +223,7 @@ IGNORED_PROCESS_COMMANDS = {' '.join(PS_CMD), '<defunct>'}
 
 
 async def get_processes_stats() -> List[ProcessInfo]:
-    result = await run_command(PS_CMD)
+    result = await run_command(PS_CMD, log_command=False)
     # Do not include `ps` in the top processes.
     ps_pid = result.pid
 
@@ -550,7 +551,7 @@ async def get_power_stats() -> PowerStats:
 
     try:
         cmd = ('dmesg',)
-        result = await run_command(cmd)
+        result = await run_command(cmd, log_command=False)
         if result.stdout and result.return_code == 0:
             under_voltage = b' Undervoltage detected' in result.stdout
             over_current = b' over-current change ' in result.stdout

wrolpi/test/test_common.py

@@ -924,7 +924,7 @@ async def test_config_lifecycle(await_switches, test_wrolpi_config):
     # Config is not imported, default values are used.
     assert config.successful_import is False
     assert config.wrol_mode is False
-    assert config.is_valid() is False
+    assert config.is_valid() is None
     assert config.version == 0
     assert config.zims_destination
 
@@ -998,7 +998,7 @@ async def test_config_lifecycle(await_switches, test_wrolpi_config):
 async def test_config_valid(async_client, test_wrolpi_config):
     # Config file does not yet exist.
     config = get_wrolpi_config()
-    assert config.is_valid() is False
+    assert config.is_valid() is None
 
     config.dump_config()
     assert config.is_valid() is True

wrolpi/test/test_config_api.py

@@ -24,7 +24,7 @@ async def test_configs_crud(async_client, test_session, test_tags_config, tag_fa
     assert response.status_code == HTTPStatus.OK
     # WROLPiConfig was not saved.
     assert 'wrolpi.yaml' in response.json['configs']
-    assert response.json['configs']['wrolpi.yaml']['valid'] is False
+    assert response.json['configs']['wrolpi.yaml']['valid'] is None
     assert response.json['configs']['wrolpi.yaml']['successful_import'] is False
     # Tags was saved, and is valid.
     assert 'tags.yaml' in response.json['configs']

wrolpi/vars.py

@@ -115,3 +115,5 @@ IS_MACOS = os.uname()[0] == 'Darwin'
 
 SIMULTANEOUS_DOWNLOAD_DOMAINS = int(os.environ.get('SIMULTANEOUS_DOWNLOAD_DOMAINS',
                                                    2 if IS_RPI else 4))
+
+WROLPI_HOME = pathlib.Path(os.environ.get('WROLPI_HOME') or '/home/wrolpi')