From 45167e4f32d65eb52a1f094229c138e5a800fc9c Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 6 Aug 2024 18:13:21 -0600 Subject: [PATCH] A Channel can be tagged with a single Tag. The Channel can be moved into a Tag Directory according to the config. --- alembic/versions/d4e446637f18_channel_tag.py | 33 ++++++ app/src/Tags.js | 4 +- app/src/api.js | 26 +++++ app/src/components/Channels.js | 106 +++++++++++++++++-- app/src/hooks/customHooks.js | 4 + main.py | 4 +- modules/archive/lib.py | 5 +- modules/archive/models.py | 21 +++- modules/archive/test/test_lib.py | 5 +- modules/archive/test/test_models.py | 27 +++++ modules/map/lib.py | 2 +- modules/map/test/test_lib.py | 2 +- modules/videos/channel/api.py | 40 ++++++- modules/videos/channel/lib.py | 52 +++++++-- modules/videos/channel/test/test_api.py | 97 ++++++++++++++++- modules/videos/channel/test/test_models.py | 17 +-- modules/videos/common.py | 5 +- modules/videos/conftest.py | 20 ++-- modules/videos/downloader.py | 9 +- modules/videos/errors.py | 6 ++ modules/videos/lib.py | 50 ++++++++- modules/videos/models.py | 74 ++++++++++--- modules/videos/schema.py | 15 +++ modules/videos/test/test_common.py | 2 +- modules/videos/test/test_downloader.py | 4 +- modules/videos/test/test_lib.py | 35 +++++- modules/videos/test/test_models.py | 26 +++++ modules/videos/video/lib.py | 2 +- modules/zim/lib.py | 2 +- modules/zim/test/test_lib.py | 2 +- requirements.txt | 2 +- scripts/singlefile_to_archive.py | 4 +- wrolpi/common.py | 62 ++++++----- wrolpi/conftest.py | 3 + wrolpi/downloader.py | 19 ++-- wrolpi/root_api.py | 32 +++--- wrolpi/schema.py | 8 +- wrolpi/tags.py | 1 - wrolpi/test/test_root_api.py | 28 ++--- 39 files changed, 691 insertions(+), 165 deletions(-) create mode 100644 alembic/versions/d4e446637f18_channel_tag.py create mode 100644 modules/archive/test/test_models.py diff --git a/alembic/versions/d4e446637f18_channel_tag.py b/alembic/versions/d4e446637f18_channel_tag.py new file mode 100644 index 00000000..7a99eefd --- /dev/null +++ b/alembic/versions/d4e446637f18_channel_tag.py @@ -0,0 +1,33 @@ +"""Channel tag + +Revision ID: d4e446637f18 +Revises: 2bfb3b50b178 +Create Date: 2024-08-03 17:49:17.133086 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'd4e446637f18' +down_revision = '2bfb3b50b178' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE channel ADD COLUMN tag_id INTEGER REFERENCES tag(id)') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS tag_id') diff --git a/app/src/Tags.js b/app/src/Tags.js index 79a59357..2e7736b1 100644 --- a/app/src/Tags.js +++ b/app/src/Tags.js @@ -131,8 +131,8 @@ export function useTags() { } - const SingleTag = ({name}) => { - return + const SingleTag = ({name, ...props}) => { + return } useEffect(() => { diff --git a/app/src/api.js b/app/src/api.js index 9a9d3e2b..c18b22fd 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -112,6 +112,32 @@ export async function getChannel(id) { return (await response.json())['channel']; } +export async function tagChannel(channelId, tagName, directory) { + const body = { + tag_name: tagName, + } + if (directory) { + body['directory'] = directory; + } + const response = await apiPost(`${VIDEOS_API}/channels/${channelId}/tag`, body); + if (!response.ok) { + toast({ + type: 'error', + title: 'Unexpected server response', + description: 'Failed to tag channel. See server logs.', + time: 5000, + }); + } +} + +export async function tagChannelInfo(channelId, tagName) { + const body = {tag_name: tagName} + const response = await apiPost(`${VIDEOS_API}/channels/${channelId}/tag_info`, body); + if (response.ok) { + return (await response.json()).videos_destination; + } +} + export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline) { // Build a search query to retrieve a list of videos from the API offset = parseInt(offset || 0); diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 93fa4516..2390f342 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -1,11 +1,13 @@ import React, {useState} from "react"; -import {Dimmer, Dropdown, Grid, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react"; +import {Dimmer, Dropdown, Grid, Label, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react"; import { createChannel, createChannelDownload, deleteChannel, deleteDownload, refreshChannel, + tagChannel, + tagChannelInfo, updateChannel, updateChannelDownload } from "../api"; @@ -39,7 +41,9 @@ import { Header, Loader, Modal, + ModalActions, ModalContent, + ModalHeader, Segment, Statistic } from "./Theme"; @@ -47,7 +51,7 @@ import {Media, ThemeContext} from "../contexts/contexts"; import {SortableTable} from "./SortableTable"; import {toast} from "react-semantic-toasts-2"; import {RecurringDownloadsTable} from "./admin/Downloads"; -import {TagsSelector} from "../Tags"; +import {TagsContext, TagsSelector} from "../Tags"; function ChannelStatistics({statistics}) { @@ -85,22 +89,31 @@ function ChannelPage({create, header}) { const [messageHeader, setMessageHeader] = useState(); const [messageContent, setMessageContent] = useState(); const [downloadModalOpen, setDownloadModalOpen] = useState(false); - const [rssDownloadOpen, setRssDownloadOpen] = useState(false); const navigate = useNavigate(); const {channelId} = useParams(); const {channel, changeValue, fetchChannel} = useChannel(channelId); + const {SingleTag} = React.useContext(TagsContext); - let title; - if (channel && channel.name) { - title = channel.name; - } - useTitle(title); + const [tagModalOpen, setTagModalOpen] = useState(false); + const [newTagName, setNewTagName] = useState(channel ? channel.tag_name : null); + const [moveToTagDirectory, setMoveToTagDirectory] = useState(true); + const [newTagDirectory, setNewTagDirectory] = useState(''); - if (!channel) { + useTitle(_.isEmpty(channel) ? null : `${channel.name} Channel`); + + React.useEffect(() => { + console.debug(channel); + if (!_.isEmpty(channel)) { + setNewTagName(channel.tag_name); + } + }, [JSON.stringify(channel)]); + + if (_.isEmpty(channel)) { return ; } + const handleCheckbox = (e, {name, checked}) => { if (e) { e.preventDefault(); @@ -247,6 +260,64 @@ function ChannelPage({create, header}) { setDownloadModalOpen(false); } + const handleTagModalOpen = (e) => { + if (e) { + e.preventDefault(); + } + setTagModalOpen(true); + } + + const handleTagChannel = async () => { + try { + await tagChannel(channelId, newTagName, moveToTagDirectory ? newTagDirectory : null); + setTagModalOpen(false); + } catch (e) { + console.error('Failed to tag channel', e); + } finally { + await fetchChannel(); + } + } + + const handleTagSelect = async (newTagName_) => { + setNewTagName(newTagName_); + try { + const videosDestination = await tagChannelInfo(channelId, newTagName_); + setNewTagDirectory(videosDestination); + } catch (e) { + console.error('Failed to tag channel', e); + } + } + + const tagModal = setTagModalOpen(false)} + closeIcon + > + {channel.tag_name ? 'Modify Tag' : 'Add Tag'} + + handleTagSelect(null)} + /> +
+ + {newTagDirectory && } + + + + + + ; + return <> {!create && @@ -324,6 +395,8 @@ function ChannelPage({create, header}) { + {channel.tag_name && } + Refresh } + + {tagModal} {channel.name} + + {channel.tag_name ? : null} + {channel.video_count} @@ -613,6 +697,8 @@ function ChannelRow({channel}) { } function MobileChannelRow({channel}) { + const {SingleTag} = React.useContext(TagsContext); + const editTo = `/videos/channel/${channel.id}/edit`; const videosTo = `/videos/channel/${channel.id}/video`; return @@ -621,6 +707,7 @@ function MobileChannelRow({channel}) {

{channel.name}

+ {channel.tag_name ? : null}

Videos: {channel.video_count} @@ -669,6 +756,7 @@ export function ChannelsPage() { const headers = [ {key: 'name', text: 'Name', sortBy: 'name', width: 8}, + {key: 'tag', text: 'Tag', sortBy: 'tag', width: 2}, {key: 'video_count', text: 'Videos', sortBy: 'video_count', width: 2}, {key: 'download_frequency', text: 'Download Frequency', sortBy: 'minimum_frequency', width: 2}, {key: 'size', text: 'Size', sortBy: 'size', width: 2}, diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index beb40eae..c8368885 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -734,6 +734,10 @@ export const useDirectories = (defaultDirectory) => { fetchDirectories(); }, [directory, defaultDirectory]); + useEffect(() => { + setDirectory(defaultDirectory); + }, [defaultDirectory]) + return {directory, directories, setDirectory, exists, isDir, isFile}; } diff --git a/main.py b/main.py index 1e1afc92..6778adf2 100755 --- a/main.py +++ b/main.py @@ -322,9 +322,9 @@ async def perpetual_have_internet_worker(): # Check hourly once we have internet. await asyncio.sleep(float(Seconds.hour)) else: - # Check every minute until the internet is back. + # Check more often until the internet is back. flags.have_internet.clear() - await asyncio.sleep(float(Seconds.minute)) + await asyncio.sleep(10) except Exception as e: logger.error('Failed to check if internet is up', exc_info=e) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 3131072d..45992410 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -48,7 +48,10 @@ class ArchiveFiles: def get_archive_directory() -> pathlib.Path: - archive_directory = get_media_directory() / get_wrolpi_config().archive_directory + archive_destination = get_wrolpi_config().archive_destination + variables = dict(domain='', year='', month='', day='') + archive_destination = archive_destination % variables + archive_directory = get_media_directory() / archive_destination return archive_directory diff --git a/modules/archive/models.py b/modules/archive/models.py index 8c41c0c4..05ea88a0 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -9,7 +9,8 @@ from sqlalchemy.orm import relationship, Session, validates from sqlalchemy.orm.collections import InstrumentedList from wrolpi import dates -from wrolpi.common import ModelHelper, Base, logger, get_title_from_html +from wrolpi.common import ModelHelper, Base, logger, get_title_from_html, get_wrolpi_config, get_media_directory +from wrolpi.dates import now from wrolpi.db import optional_session from wrolpi.errors import UnknownArchive from wrolpi.files.models import FileGroup @@ -342,3 +343,21 @@ class Domain(Base, ModelHelper): if len(value.split('.')) < 2: raise ValueError(f'Domain must contain at least one "." domain={repr(value)}') return value + + @property + def download_directory(self) -> pathlib.Path: + archive_destination = get_wrolpi_config().archive_destination + + now_ = now() + variables = dict( + domain=self.domain, + year=now_.year, + month=now_.month, + day=now_.day, + ) + archive_destination = archive_destination % variables + archive_destination = pathlib.Path(archive_destination.lstrip('/')) + if not archive_destination.is_absolute(): + archive_destination = get_media_directory() / archive_destination + + return archive_destination diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index d9634c30..ceb254b6 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -820,12 +820,13 @@ def test_get_url_from_singlefile(): assert lib.get_url_from_singlefile(SINGLEFILE_EXAMPLE_1) == 'https://www.example.com' -def test_get_custom_archive_directory(test_directory, test_config): +@pytest.mark.asyncio +async def test_get_custom_archive_directory(test_async_client, test_directory, test_config): """Custom directory can be used for archive directory.""" # Default location. assert lib.get_archive_directory() == (test_directory / 'archive') - get_wrolpi_config().archive_directory = 'custom/archives' + get_wrolpi_config().archive_destination = 'custom/archives' assert lib.get_archive_directory() == (test_directory / 'custom/archives') assert lib.get_domain_directory('https://example.com') == (test_directory / 'custom/archives/example.com') diff --git a/modules/archive/test/test_models.py b/modules/archive/test/test_models.py new file mode 100644 index 00000000..c580d6d1 --- /dev/null +++ b/modules/archive/test/test_models.py @@ -0,0 +1,27 @@ +from datetime import datetime + +import pytest + +from modules.archive import Domain +from wrolpi.common import get_wrolpi_config + + +@pytest.mark.asyncio +async def test_archive_download_destination(test_async_client, test_session, test_directory, archive_factory, fake_now): + fake_now(datetime(2000, 1, 2)) + + wrolpi_config = get_wrolpi_config() + + archive_factory(domain='wrolpi.org') + domain = test_session.query(Domain).one() + + # Test the default download directory. + assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org') + + # Year of download is supported. + wrolpi_config.archive_destination = 'archives/%(domain)s/%(year)s' + assert str(domain.download_directory) == str(test_directory / 'archives/wrolpi.org/2000') + + # More download date is supported. + wrolpi_config.archive_destination = 'archive/%(domain)s/%(year)s/%(month)s/%(day)s' + assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org/2000/1/2') diff --git a/modules/map/lib.py b/modules/map/lib.py index 0107b430..f72ba914 100644 --- a/modules/map/lib.py +++ b/modules/map/lib.py @@ -19,7 +19,7 @@ logger = logger.getChild(__name__) def get_map_directory() -> Path: - map_directory = get_media_directory() / get_wrolpi_config().map_directory + map_directory = get_media_directory() / get_wrolpi_config().map_destination if not map_directory.is_dir(): map_directory.mkdir(parents=True) return map_directory diff --git a/modules/map/test/test_lib.py b/modules/map/test/test_lib.py index 8e31d4cf..a1ed847b 100644 --- a/modules/map/test/test_lib.py +++ b/modules/map/test/test_lib.py @@ -107,7 +107,7 @@ def test_get_custom_map_directory(test_directory, test_config): # Default location. assert lib.get_map_directory() == (test_directory / 'map') - get_wrolpi_config().map_directory = 'custom/deep/map/directory' + get_wrolpi_config().map_destination = 'custom/deep/map/directory' assert lib.get_map_directory() == (test_directory / 'custom/deep/map/directory') assert (test_directory / 'custom/deep/map/directory').is_dir() diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index ea259a0e..4bc5ade3 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -17,7 +17,7 @@ from wrolpi.vars import PYTEST from . import lib from .. import schema from ..errors import UnknownChannel -from ..lib import save_channels_config +from ..lib import save_channels_config, format_videos_destination from ..models import Channel channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels') @@ -136,3 +136,41 @@ async def channel_download(_: Request, channel_id: int, body: schema.ChannelDown # Warn the user that downloads are disabled. Events.send_downloads_disabled('Channel download created. But downloads are disabled.') return response.empty() + + +@channel_bp.post('//tag_info') +@openapi.definition( + description='Get Channel Tag Info', + body=schema.ChannelTagInfoRequest, +) +@validate(schema.ChannelTagInfoRequest) +@run_after(save_channels_config) +@wrol_mode_check +async def channel_tag_info(_: Request, channel_id: int, body: schema.ChannelTagInfoRequest): + from wrolpi.tags import Tag + channel_name = Channel.find_by_id(channel_id).name + tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None + videos_destination = format_videos_destination(channel_name, tag_name) + ret = dict(videos_destination=videos_destination) + return json_response(ret) + + +@channel_bp.post('//tag') +@openapi.definition( + description='Tag Channel', + body=schema.ChannelTagRequest, +) +@openapi.response(HTTPStatus.NO_CONTENT) +@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) +@validate(schema.ChannelTagRequest) +@run_after(save_channels_config) +@wrol_mode_check +async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagRequest): + directory = None + if body.directory: + directory = pathlib.Path(body.directory) + directory = get_media_directory() / directory + + await lib.tag_channel(body.tag_name, directory, channel_id) + + return response.raw('', HTTPStatus.NO_CONTENT) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index bba55415..055658b4 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -1,9 +1,9 @@ +import pathlib from pathlib import Path from typing import List, Dict, Union from sqlalchemy import or_, func, desc, asc from sqlalchemy.orm import Session -from sqlalchemy.orm.exc import NoResultFound from wrolpi.common import run_after, logger, \ get_media_directory @@ -12,7 +12,7 @@ from wrolpi.downloader import save_downloads_config, download_manager from wrolpi.errors import UnknownDirectory, APIError, ValidationError from .. import schema from ..common import check_for_channel_conflicts -from ..errors import UnknownChannel +from ..errors import UnknownChannel, ChannelDirectoryConflict from ..lib import save_channels_config from ..models import Channel, Video @@ -28,7 +28,7 @@ async def get_minimal_channels() -> List[dict]: # one that will consume the most resources. stmt = ''' SELECT - c.id, name, directory, c.url, + c.id, c.name AS "name", directory, c.url, t.name AS "tag_name", COUNT(v.id) as video_count, SUM(fg.size)::BIGINT AS size, ( @@ -40,7 +40,8 @@ async def get_minimal_channels() -> List[dict]: channel AS c LEFT JOIN video v on c.id = v.channel_id LEFT JOIN file_group fg on fg.id = v.file_group_id - GROUP BY 1, 2, 3, 4 + LEFT JOIN tag t ON t.id = c.tag_id + GROUP BY 1, 2, 3, 4, 5 ''' curs.execute(stmt) channels = sorted([dict(i) for i in curs.fetchall()], key=lambda i: i['name'].lower()) @@ -115,10 +116,7 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No @optional_session def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel: """Update a Channel's DB record""" - try: - channel: Channel = session.query(Channel).filter_by(id=channel_id).one() - except NoResultFound: - raise UnknownChannel() + channel = Channel.find_by_id(channel_id, session) # Only update directory if it was empty if data.directory and not channel.directory: @@ -237,8 +235,9 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f Channel.find_by_id(channel_id, session=session) download = download_manager.get_download(session, id_=download_id) download.url = url - download.frequency = download.frequency or frequency + download.frequency = frequency download.settings = settings + session.commit() download_manager.remove_from_skip_list(url) @@ -246,3 +245,38 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f await save_downloads_config() return download + + +@optional_session +async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id: int, session: Session = None): + """Add a Tag to a Channel, or remove a Tag from a Channel if no `tag_name` is provided. + + Move the Channel to the new directory, if provided.""" + from wrolpi.tags import Tag + + channel = Channel.find_by_id(channel_id, session=session) + + if tag_name: + tag = Tag.find_by_name(tag_name, session=session) + channel.tag_id = tag.id + channel.tag = tag + else: + channel.tag = channel.tag_id = None + + session.flush([channel, ]) + + # Do not move Channel files into a directory with files. + if directory and directory.is_dir() and next(directory.iterdir(), None): + raise ChannelDirectoryConflict('Channel directory already exists and is not empty') + + # Only move Channel when requested. + if directory: + # Move to newly defined directory only if necessary. + logger.info(f'Tagging {channel} with {tag_name} moving to {directory} from {channel.directory}') + directory.mkdir(parents=True, exist_ok=True) + if channel.directory != directory: + await channel.move(directory, session) + else: + logger.info(f'Tagging {channel} with {tag_name}') + + session.commit() diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index ea8fdd60..c615bb4f 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -8,8 +8,11 @@ from http import HTTPStatus import mock import pytest +from modules.videos.common import get_videos_directory from modules.videos.downloader import ChannelDownloader -from modules.videos.models import Channel +from modules.videos.lib import save_channels_config, get_channels_config +from modules.videos.models import Channel, Video +from wrolpi.common import get_relative_to_media_directory, walk from wrolpi.dates import now from wrolpi.db import get_db_session from wrolpi.downloader import download_manager, Download, DownloadResult @@ -412,7 +415,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto @pytest.mark.asyncio -async def test_create_channel_download(test_async_client, test_session, simple_channel, tag_factory): +async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory): tag = tag_factory() # Create Download. @@ -452,3 +455,93 @@ async def test_create_channel_download(test_async_client, test_session, simple_c assert response.status_code == HTTPStatus.NO_CONTENT assert test_session.query(Download).count() == 1 assert test_session.query(Channel).count() == 1 + + +@pytest.mark.asyncio +async def test_tag_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory, video_factory, + test_channels_config, test_download_manager, test_downloader): + """A single Tag can be applied to a Channel.""" + # Create channel directory in the usual videos directory. + videos_directory = get_videos_directory() + channel = channel_factory(name='Channel Name', download_frequency=120) + channel_directory = channel.directory + tag = tag_factory('Tag Name') + v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id) + # Create recurring download which uses the Channel's directory. + download = download_manager.recurring_download('https://example.com/1', 60, test_downloader.name, + settings={ + 'destination': str(test_directory / 'videos/Channel Name')}) + test_session.commit() + save_channels_config() + assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Channel Name') + # Channel download downloads into the Channel's directory. + assert channel.downloads[0].settings['destination'] == str(test_directory / 'videos/Channel Name') + # Make extra file in the Channel's directory, it should be moved. + (channel_directory / 'extra file.txt').write_text('extra file contents') + # Channel directory is in the Videos directory. + assert channel.directory == channel_directory + # Videos are in the Channel's Directory. + assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4' + assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4' + video_files = [i for i in walk(videos_directory) if i.is_file()] + assert len(video_files) == 3 + + body = dict(tag_name=tag.name, directory='videos/Tag Name/Channel Name') + request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT + + channel = test_session.query(Channel).one() + assert channel.tag_name == tag.name + # Channel was moved to Tag's directory, old directory was removed + new_channel_directory = test_directory / 'videos/Tag Name/Channel Name' + assert channel.directory == new_channel_directory, 'Channel directory should have been changed.' + assert new_channel_directory.is_dir(), 'Channel should have been moved.' + assert next(new_channel_directory.iterdir(), None), 'Channel videos should have been moved.' + assert not channel_directory.exists(), 'Old Channel directory should have been deleted.' + # Channel download goes into the Channel's directory. + d1, d2 = test_session.query(Download).all() + assert d1.settings['destination'] == str(test_directory / 'videos/Tag Name/Channel Name'), \ + f'{d1} was not moved' + assert d2.settings['destination'] == str(test_directory / 'videos/Tag Name/Channel Name'), \ + f'{d2} was not moved' + + # Videos were moved + v1, v2 = test_session.query(Video).order_by(Video.id).all() + assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Tag Name/Channel Name/video1.mp4' + assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Tag Name/Channel Name/video2.mp4' + # Extra file was also moved. + assert (new_channel_directory / 'extra file.txt').read_text() == 'extra file contents' + # No new files were created. + assert len([i for i in walk(videos_directory) if i.is_file()]) == 3 + + assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Tag Name/Channel Name') + + # Remove tag, Channel/Videos should be moved back. + body = dict(tag_name=None, directory='videos/Channel Name') + request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode() + channel = test_session.query(Channel).one() + assert channel.tag_name is None + assert str(channel.directory) == str(test_directory / 'videos/Channel Name') + v1, v2 = test_session.query(Video).order_by(Video.id).all() + assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4' + assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4' + assert not (test_directory / 'videos/Tag Name/Channel Name').exists() + # Downloads are moved back. + d1, d2 = test_session.query(Download).all() + assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved' + assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved' + + # Channel can be Tagged, but not move directories. + body = dict(tag_name=tag.name) + request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode() + channel = test_session.query(Channel).one() + assert channel.tag_name == 'Tag Name' + assert str(channel.directory) == str(test_directory / 'videos/Channel Name') + assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4' + assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4' + # Downloads were not changed. + d1, d2 = test_session.query(Download).all() + assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved' + assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved' diff --git a/modules/videos/channel/test/test_models.py b/modules/videos/channel/test/test_models.py index 3668db42..80deab14 100644 --- a/modules/videos/channel/test/test_models.py +++ b/modules/videos/channel/test/test_models.py @@ -3,10 +3,9 @@ from http import HTTPStatus import pytest -from modules.videos.downloader import VideoDownloader from modules.videos.models import Channel from modules.videos.models import Video -from wrolpi.downloader import Download, DownloadFrequency, download_manager, RSSDownloader +from wrolpi.downloader import Download, DownloadFrequency def test_delete_channel_no_url(test_session, test_client, channel_factory): @@ -102,17 +101,3 @@ async def test_channel_download_relationships(test_session, download_channel): # Delete the Channel, and any relationships. download_channel.delete_with_videos() assert test_session.query(Download).count() == 0 - - -@pytest.mark.asyncio() -async def test_create_channel_download(test_session, simple_channel): - """A Download will be related to a Channel if a Download is created in the Channel's directory""" - test_session.flush() - - settings = dict(destination=str(simple_channel.directory)) - download_manager.recurring_download('https://example.com/2', 1, RSSDownloader.name, test_session, - sub_downloader_name=VideoDownloader.name, settings=settings) - download = test_session.query(Download).one() - assert download.settings['destination'] == str(simple_channel.directory) - download = test_session.query(Download).one() - assert download.channel == simple_channel diff --git a/modules/videos/common.py b/modules/videos/common.py index c63e0745..8a4307ef 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -35,8 +35,9 @@ class ConfigError(Exception): def get_videos_directory() -> pathlib.Path: """Get the "videos" directory in the media directory. Make it if it does not exist.""" - config = get_wrolpi_config() - directory = get_media_directory() / config.videos_directory + from modules.videos.lib import format_videos_destination + videos_destination = format_videos_destination() + directory = get_media_directory() / videos_destination if not directory.is_dir(): directory.mkdir(parents=True) return directory diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 0b417f96..0e2f7240 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -11,12 +11,11 @@ import pytest from PIL import Image from modules.videos.downloader import VideoDownloader, ChannelDownloader -from modules.videos.lib import set_test_channels_config, set_test_downloader_config +from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination from modules.videos.models import Channel, Video from wrolpi.api_utils import api_app from wrolpi.downloader import DownloadFrequency, DownloadManager from wrolpi.files.models import FileGroup -from wrolpi.tags import Tag from wrolpi.vars import PROJECT_DIR @@ -36,25 +35,25 @@ def simple_channel(test_session, test_directory) -> Channel: @pytest.fixture def channel_factory(test_session, test_directory): """Create a random Channel with a directory, and Download.""" + from wrolpi.tags import Tag def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None, - directory: pathlib.Path = None): + directory: pathlib.Path = None, tag_name: str = None): name = name or str(uuid4()) - directory = directory or test_directory / name - if not directory.is_dir(): - directory.mkdir() + tag = Tag.get_by_name(tag_name) if tag_name else None channel = Channel( - directory=directory, # noqa name=name, url=url or f'https://example.com/{name}', source_id=source_id, + tag=tag, ) + channel.directory = directory or format_videos_destination(name, tag, channel.url) + channel.directory.mkdir(exist_ok=True, parents=True) test_session.add(channel) test_session.flush([channel]) assert channel.id and channel.url if download_frequency: - download = channel.get_or_create_download(channel.url, 60) - download.frequency = download_frequency + download = channel.get_or_create_download(channel.url, download_frequency) assert download.url == channel.url assert channel.downloads test_session.commit() @@ -106,8 +105,9 @@ def video_factory(test_session, test_directory): path = pathlib.Path(with_video_file) if not isinstance(with_video_file, pathlib.Path) else with_video_file elif channel_id: # Put the video in its Channel's directory. - channel = test_session.query(Channel).filter_by(id=channel_id).one() + channel = Channel.find_by_id(channel_id) path = (channel.directory or test_directory) / f'{title}.mp4' + path.parent.mkdir(exist_ok=True, parents=True) else: # Put any video not in a Channel and without a specified file in the NO CHANNEL directory. (test_directory / 'videos/NO CHANNEL').mkdir(exist_ok=True, parents=True) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 95346e89..2979846f 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -264,7 +264,7 @@ class VideoDownloader(Downloader, ABC): channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id') channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url') channel = None - if channel_name or channel_source_id: + if channel_name or channel_source_id or channel_url: # Try to find the channel via info_json from yt-dlp. try: channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name) @@ -278,7 +278,7 @@ class VideoDownloader(Downloader, ABC): settings = download.settings or dict() destination = settings.get('destination') if not channel and destination: - # Destination may override the real Channel. + # Destination may find Channel if not already found. try: channel = get_channel(directory=destination, return_dict=False) found_channel = 'download_settings_directory' @@ -290,7 +290,7 @@ class VideoDownloader(Downloader, ABC): local_channel_id = settings.get('channel_id') channel_url = settings.get('channel_url') if not channel and (local_channel_id or channel_url): - # Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download. + # Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download. logger.info(f'Using download.settings to find channel') try: channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False) @@ -320,11 +320,12 @@ class VideoDownloader(Downloader, ABC): logger.debug(f'Downloading {url} to destination from settings') elif channel: out_dir = channel_directory - logger.debug(f'Downloading {url} to channel directory') + logger.debug(f'Downloading {url} to channel directory: {channel_directory}') else: # Download to the default directory if this video has no channel. out_dir = get_no_channel_directory() logger.debug(f'Downloading {url} to default directory') + out_dir.mkdir(exist_ok=True, parents=True) logs = None # noqa diff --git a/modules/videos/errors.py b/modules/videos/errors.py index 18633148..a42d741f 100644 --- a/modules/videos/errors.py +++ b/modules/videos/errors.py @@ -15,6 +15,12 @@ class UnknownChannel(APIError): status_code = HTTPStatus.NOT_FOUND +class InvalidChannel(APIError): + code = 'INVALID_CHANNEL' + message = 'The channel is not supported' + status_code = HTTPStatus.BAD_REQUEST + + class ChannelNameConflict(APIError): code = 'CHANNEL_NAME_CONFLICT' message = 'The channel name is already taken.' diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 2cd84f16..265031d5 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -3,27 +3,30 @@ import dataclasses import html import pathlib import re -from typing import Tuple, Optional, Generator +from typing import Generator +from typing import Tuple, Optional import pytz from sqlalchemy import or_ from sqlalchemy.orm import Session from yt_dlp import YoutubeDL +from modules.videos.models import Video from wrolpi import before_startup, dates, flags from wrolpi.captions import extract_captions -from wrolpi.common import ConfigFile, get_media_directory, register_refresh_cleanup, limit_concurrent +from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \ + escape_file_name, get_media_directory from wrolpi.dates import Seconds, from_timestamp from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.downloader import Download from wrolpi.errors import UnknownDirectory from wrolpi.files.lib import split_path_stem_and_suffix -from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR +from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST from .common import is_valid_poster, convert_image, \ generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \ get_video_duration from .errors import UnknownChannel -from .models import Channel, Video +from .models import Channel logger = logger.getChild(__name__) @@ -377,7 +380,6 @@ channel_import_logger = logger.getChild('channel_import') @before_startup -@register_refresh_cleanup @limit_concurrent(1) def import_channels_config(): """Import channel settings to the DB. Existing channels will be updated.""" @@ -645,3 +647,41 @@ def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path ''') results = (pathlib.Path(j['path']) for i in curs.fetchall() for j in i[0]) yield from results + + +def format_videos_destination(channel_name: str = None, channel_tag: str = None, channel_url: str = None) \ + -> pathlib.Path: + """Return the directory where Videos should be downloaded according to the WROLPi Config. + + @warning: Directory may or may not exist.""" + videos_destination = get_wrolpi_config().videos_destination + + channel_domain = '' + if channel_url: + try: + channel_domain = extract_domain(channel_url) + except Exception as e: + logger.error(f'Failed to extract domain from Channel URL: {channel_url}', exc_info=e) + if PYTEST: + raise + + if channel_name and not isinstance(channel_name, str): + raise RuntimeError('channel name must be string') + if channel_tag and not isinstance(channel_tag, str): + raise RuntimeError('channel tag must be string') + + name = escape_file_name(channel_name) if channel_name else '' + variables = dict( + channel_name=name, + channel_tag=channel_tag or '', + channel_domain=channel_domain, + ) + + try: + videos_destination = videos_destination % variables + except KeyError as e: + raise FileNotFoundError(f'Cannot download to the defined "videos_destination"') from e + + videos_destination = get_media_directory() / videos_destination.lstrip('/') + + return videos_destination diff --git a/modules/videos/models.py b/modules/videos/models.py index 6ae9ed9e..1b9b598e 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -1,6 +1,5 @@ import json import pathlib -from pathlib import Path from typing import Optional, Dict, List from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger @@ -11,12 +10,12 @@ from modules.videos.errors import UnknownVideo, UnknownChannel from wrolpi.captions import read_captions from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file from wrolpi.db import get_db_curs, get_db_session, optional_session -from wrolpi.downloader import Download, download_manager +from wrolpi.downloader import Download, download_manager, save_downloads_config from wrolpi.errors import InvalidDownload -from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix +from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, move from wrolpi.files.models import FileGroup from wrolpi.media_path import MediaPathType -from wrolpi.tags import Tag, TagFile +from wrolpi.tags import Tag, TagFile, save_tags_config from wrolpi.vars import PYTEST, DELETED_VIDEO_KEYS logger = logger.getChild(__name__) @@ -533,10 +532,16 @@ class Channel(ModelHelper, Base): videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id') downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id') + tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE')) + tag = relationship('Tag') def __repr__(self): return f'' + @property + def tag_name(self) -> str | None: + return self.tag.name if self.tag else None + def delete_with_videos(self): """Delete all Video records (but not video files) related to this Channel. Then delete the Channel.""" session = Session.object_session(self) @@ -560,12 +565,20 @@ class Channel(ModelHelper, Base): channel_id = self.id data = data.copy() + # Remove unused values. + data.pop('move_to_tag_directory', None) + # URL should not be empty string. if 'url' in data: data['url'] = data['url'] or None + session: Session = Session.object_session(self) + downloads = data.pop('downloads', []) + if tag_name := data.pop('tag_name', None): + self.tag = Tag.find_by_name(tag_name, session) + for key, value in data.items(): setattr(self, key, value) @@ -575,8 +588,6 @@ class Channel(ModelHelper, Base): elif isinstance(self.directory, str) and not pathlib.Path(self.directory).is_absolute(): self.directory = get_media_directory() / self.directory - session: Session = Session.object_session(self) - for download in downloads: url = None frequency = None @@ -608,11 +619,12 @@ class Channel(ModelHelper, Base): config = dict( calculate_duration=self.calculate_duration, directory=str(self.directory), + downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads], generate_posters=self.generate_posters, name=self.name, source_id=self.source_id, + tag_name=self.tag_name, url=self.url or None, - downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads] ) return config @@ -627,16 +639,18 @@ class Channel(ModelHelper, Base): def __json__(self) -> dict: d = dict( - downloads=self.downloads, directory=self.directory, + downloads=self.downloads, id=self.id, name=self.name, + tag_name=self.tag_name, url=self.url, ) return d def dict(self, with_statistics: bool = False, with_downloads: bool = True) -> dict: d = super(Channel, self).dict() + d['tag_name'] = self.tag_name d['directory'] = \ self.directory.relative_to(get_media_directory()) if self.directory else None if with_statistics: @@ -706,10 +720,9 @@ class Channel(ModelHelper, Base): """Find a Channel with the provided id, raises an exception when no Channel is found. @raise UnknownChannel: if the channel can not be found""" - channel = Channel.get_by_id(id_, session=session) - if not channel: - raise UnknownChannel(f'Cannot find channel with id {id_}') - return channel + if channel := Channel.get_by_id(id_, session=session): + return channel + raise UnknownChannel(f'Cannot find channel with id {id_}') def get_video_entry_by_id(self, video_source_id: str) -> Optional[Dict]: """Search my info_json for the entry with the provided id.""" @@ -758,3 +771,40 @@ class Channel(ModelHelper, Base): raise RuntimeError('Must provide URL to get Channel') channel = session.query(Channel).filter_by(url=url).one_or_none() return channel + + async def move(self, directory: pathlib.Path, session: Session): + """Move the files of this Channel into a new directory.""" + if not directory.is_dir(): + raise FileNotFoundError(f'Destination directory does not exist: {directory}') + if self.directory and not next(self.directory.iterdir(), None): + raise FileExistsError(f'Refusing to move Channel when it has no files.') + + old_directory = self.directory + self.directory = directory + session.flush([self, ]) + + # Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's + # directory. + downloads = list(self.downloads) + downloads.extend(list( + session.query(Download).filter(Download.settings.contains({'destination': str(old_directory)})) + )) + for download in downloads: + # Get new copy of `settings` to avoid bug where it wouldn't update. + settings = download.settings.copy() if download.settings else dict() + if settings.get('destination'): + download.settings = {**settings, 'destination': str(directory)} + + session.commit() + + # Save configs before move, this is because move imports configs. + from modules.videos.lib import save_channels_config + await save_downloads_config(session) + save_channels_config(session) + save_tags_config(session) + + # Move the contents of the Channel directory into the destination directory. + await move(directory, *list(old_directory.iterdir())) + + if old_directory.exists() and not next(old_directory.iterdir(), None): + old_directory.rmdir() diff --git a/modules/videos/schema.py b/modules/videos/schema.py index cb2b7f19..4760149d 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -37,6 +37,21 @@ class ChannelPutRequest: self.url = self.url.strip() if self.url else None +@dataclass +class ChannelTagRequest: + tag_name: Optional[str] = None + directory: Optional[str] = None + + def __post_init__(self): + if self.directory and self.directory.startswith('/'): + raise ValidationError('Directory must be relative to media directory') + + +@dataclass +class ChannelTagInfoRequest: + tag_name: Optional[str] = None + + @dataclass class ChannelDownloadRequest: url: str diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index b5aea3c8..00044b5e 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -332,7 +332,7 @@ def test_get_videos_directory(test_directory): def test_get_custom_videos_directory(test_directory, test_config): """Custom directory can be used for videos directory.""" config = get_wrolpi_config() - config.videos_directory = 'custom/directory/videos' + config.videos_destination = 'custom/directory/videos' assert common.get_videos_directory() == (test_directory / 'custom/directory/videos') assert (test_directory / 'custom/directory/videos').is_dir() diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 4bd30bf1..d432591d 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -19,7 +19,7 @@ example_video_json = { 'categories': ['Howto & Style'], 'channel': 'channel name', 'channel_id': 'channel id', - 'channel_url': 'channel url', + 'channel_url': 'https://example.com/example_video_json_channel', 'duration': 635, 'extractor': 'youtube', 'extractor_key': 'Youtube', @@ -31,7 +31,7 @@ example_video_json = { 'upload_date': '20190707', 'uploader': 'uploader name', 'uploader_id': 'uploader id', - 'uploader_url': 'uploader url', + 'uploader_url': 'https://example.com/example_video_json_uploader', 'view_count': 406, 'webpage_url': 'webpage url', 'webpage_url_basename': 'watch', diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index fc2fee71..01907e19 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -5,8 +5,9 @@ import pytest from modules.videos import lib from modules.videos.downloader import ChannelDownloader, VideoDownloader -from modules.videos.lib import parse_video_file_name, validate_video, get_statistics +from modules.videos.lib import parse_video_file_name, validate_video, get_statistics, format_videos_destination from modules.videos.models import Video, Channel +from wrolpi.common import get_wrolpi_config from wrolpi.downloader import Download, RSSDownloader from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup @@ -20,7 +21,7 @@ from wrolpi.vars import PROJECT_DIR ('20000101foo.mp4', (None, None, None, '20000101foo')), ('something 20000101 foo.mp4', (None, None, None, 'something 20000101 foo')), ('something_20000101_foo.mp4', ('something', '20000101', None, 'foo')), - ('foo .mp4', (None, None, None, 'foo')), # Has trailing whitespace. + ('foo .mp4', (None, None, None, 'foo')), # Has trailing whitespace. ('NA_20000303_vp91w5_Bob's Pancakes.mp4', (None, '20000303', 'vp91w5', 'Bob's Pancakes')), ('NA_NA_vp91w5_Bob's Pancakes.mp4', (None, None, 'vp91w5', 'Bob's Pancakes')), ('Learning Self-Reliance_20170529_p_MzsCFkUPU_Beekeeping 2017 Part 6 - Merging Hives.mp4', @@ -216,3 +217,33 @@ def test_link_channel_and_downloads_migration(test_async_client, test_session, c assert d2b.channel_id assert d2c.url == 'https://example.com/channel2/video/1' and d2c.frequency is None assert not d2c.channel_id + + +@pytest.mark.asyncio +async def test_format_videos_destination(test_async_client, test_directory): + """Videos destination is formatted according to the WROLPiConfig.""" + wrolpi_config = get_wrolpi_config() + + # Channel directory without a tag. + wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_name)s' + assert format_videos_destination('Simple Channel', None, None) \ + == test_directory / 'videos/Simple Channel' + + # One tag can be applied to a Channel. + assert format_videos_destination('Simple Channel', 'one', None) \ + == test_directory / 'videos/one/Simple Channel' + + # Channel Domain is also supported. + wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s' + assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \ + == test_directory / 'videos/one/example.com/Simple Channel' + + # Channel name is not required. + wrolpi_config.videos_destination = '%(channel_tag)s/%(channel_domain)s' + assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \ + == test_directory / 'one/example.com' + + # Invalid `videos_destination` raises an error. + wrolpi_config.videos_destination = '%(channel_tag)s/%(channel)s' + with pytest.raises(FileNotFoundError): + assert format_videos_destination() diff --git a/modules/videos/test/test_models.py b/modules/videos/test/test_models.py index 4a678064..436d4cb6 100644 --- a/modules/videos/test/test_models.py +++ b/modules/videos/test/test_models.py @@ -1,5 +1,10 @@ +import pathlib + import pytest +from modules.videos.models import Video, Channel +from wrolpi.common import get_relative_to_media_directory, get_wrolpi_config + @pytest.mark.asyncio() async def test_ffprobe_stream_methods(simple_video): @@ -13,3 +18,24 @@ async def test_ffprobe_stream_methods(simple_video): assert simple_video.get_streams_by_codec_name('h264') assert simple_video.get_streams_by_codec_name('aac') assert simple_video.get_streams_by_codec_name('mov_text') + + +@pytest.mark.asyncio +async def test_channel_move(test_async_client, test_session, test_directory, channel_factory, video_factory): + """A Channel can be moved to another directory, any files in the Channel's directory are moved.""" + channel = channel_factory(name='Channel Name') + video = video_factory(title='Vid', channel_id=channel.id) + test_session.commit() + assert video.channel_id == channel.id + assert str(get_relative_to_media_directory(video.video_path)) == 'videos/Channel Name/Vid.mp4' + + foo = test_directory / 'foo/New Channel Directory' + foo.mkdir(parents=True) + await channel.move(foo, test_session) + assert channel.directory == foo + assert channel.directory != (test_directory / 'New Channel Directory') + video = test_session.query(Video).one() + assert str(get_relative_to_media_directory(video.video_path)) == 'foo/New Channel Directory/Vid.mp4' + assert str(video.video_path).startswith(str(channel.directory)) + # Old directory is deleted. + assert not (test_directory / 'Channel Name').exists() diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index 89ec00bf..79775c1e 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -85,7 +85,7 @@ def search_videos( joins = list() join_video = False - params = dict(search_str=search_str, offset=offset) + params = dict(search_str=search_str, offset=offset or 0) if channel_id: wheres.append('v.channel_id = %(channel_id)s') joins.append('LEFT JOIN channel c ON c.id = v.channel_id') diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 7d4140c3..4e9a3936 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -464,7 +464,7 @@ def zim_download_url_to_name(url: str) -> Tuple[str, str]: def get_zim_directory() -> pathlib.Path: - zim_directory = get_media_directory() / get_wrolpi_config().zims_directory + zim_directory = get_media_directory() / get_wrolpi_config().zims_destination return zim_directory diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 828ff465..7899f9fd 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -385,6 +385,6 @@ def test_get_custom_zims_directory(test_directory, test_config): """Custom directory can be used for zims directory.""" assert lib.get_zim_directory() == (test_directory / 'zims') - get_wrolpi_config().zims_directory = 'custom/zims' + get_wrolpi_config().zims_destination = 'custom/zims' assert lib.get_zim_directory() == (test_directory / 'custom/zims') diff --git a/requirements.txt b/requirements.txt index 0b276215..f18abcf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,4 +27,4 @@ srt==3.5.3 vininfo[cli]==1.8.0 websockets==12.0 webvtt-py==0.4.6 -yt-dlp==2024.8.1 \ No newline at end of file +yt-dlp==2024.8.6 \ No newline at end of file diff --git a/scripts/singlefile_to_archive.py b/scripts/singlefile_to_archive.py index b3bd5bd1..227e46fc 100755 --- a/scripts/singlefile_to_archive.py +++ b/scripts/singlefile_to_archive.py @@ -10,7 +10,7 @@ import tempfile from modules.archive import lib as archive_lib from wrolpi.cmd import READABILITY_BIN -from wrolpi.common import html_screenshot, format_html_string, logger +from wrolpi.common import html_screenshot, format_html_string, logger, format_html_file logger = logger.getChild('singlefile_to_archive') @@ -31,6 +31,8 @@ def singlefile_to_archive_files(singlefile: pathlib.Path): """Extract Readability files and Screenshot file next to the provided Singlefile.""" contents = singlefile.read_bytes() + format_html_file(singlefile) + readability_html_path = singlefile.with_suffix('.readability.html') readability_json_path = singlefile.with_suffix('.readability.json') readability_text_path = singlefile.with_suffix('.readability.txt') diff --git a/wrolpi/common.py b/wrolpi/common.py index 88f6b45e..2ac12c31 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -276,7 +276,10 @@ class ModelHelper: def flush(self): """A convenience function which flushes this record using its DB Session.""" - Session.object_session(self).flush([self]) + if session := Session.object_session(self): + session.flush([self]) + else: + raise RuntimeError(f'{self} is not in a session!') class tsvector(types.TypeDecorator): @@ -370,7 +373,8 @@ class ConfigFile: if config_file.is_file(): # Use the config file to get the values the user set. with config_file.open('rt') as fh: - self._config.update(yaml.load(fh, Loader=yaml.Loader)) + config_data = yaml.load(fh, Loader=yaml.Loader) + self._config.update(config_data) return self def _get_backup_filename(self): @@ -467,7 +471,7 @@ class ConfigFile: class WROLPiConfig(ConfigFile): file_name = 'wrolpi.yaml' default_config = dict( - archive_directory='archive', + archive_destination='archive/%(domain)s', download_on_startup=True, download_timeout=0, hotspot_device='wlan0', @@ -476,12 +480,12 @@ class WROLPiConfig(ConfigFile): hotspot_ssid='WROLPi', ignore_outdated_zims=False, ignored_directories=['config', 'tags'], - map_directory='map', + map_destination='map', nav_color='violet', throttle_on_startup=False, - videos_directory='videos', + videos_destination='videos/%(channel_tag)s/%(channel_name)s', wrol_mode=False, - zims_directory='zims', + zims_destination='zims', ) def get_file(self) -> Path: @@ -586,28 +590,36 @@ class WROLPiConfig(ConfigFile): self.update({'ignored_directories': value}) @property - def videos_directory(self) -> str: - return self._config['videos_directory'] + def videos_destination(self) -> str: + return self._config['videos_destination'] - @videos_directory.setter - def videos_directory(self, value: str): - self.update({'videos_directory': value}) + @videos_destination.setter + def videos_destination(self, value: str): + self.update({'videos_destination': value}) @property - def archive_directory(self) -> str: - return self._config['archive_directory'] + def archive_destination(self) -> str: + return self._config['archive_destination'] - @archive_directory.setter - def archive_directory(self, value: str): - self.update({'archive_directory': value}) + @archive_destination.setter + def archive_destination(self, value: str): + self.update({'archive_destination': value}) @property - def map_directory(self) -> str: - return self._config['map_directory'] + def map_destination(self) -> str: + return self._config['map_destination'] - @map_directory.setter - def map_directory(self, value: str): - self.update({'map_directory': value}) + @map_destination.setter + def map_destination(self, value: str): + self.update({'map_destination': value}) + + @property + def zims_destination(self) -> str: + return self._config['zims_destination'] + + @zims_destination.setter + def zims_destination(self, value: str): + self.update({'zims_destination': value}) @property def nav_color(self) -> str: @@ -617,14 +629,6 @@ class WROLPiConfig(ConfigFile): def nav_color(self, value: str): self.update({'nav_color': value}) - @property - def zims_directory(self) -> str: - return self._config['zims_directory'] - - @zims_directory.setter - def zims_directory(self, value: str): - self.update({'zims_directory': value}) - WROLPI_CONFIG: WROLPiConfig = WROLPiConfig() TEST_WROLPI_CONFIG: WROLPiConfig = None diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index a0027892..2492e107 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -36,6 +36,7 @@ import wrolpi.root_api # noqa from wrolpi import flags from wrolpi.api_utils import api_app from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode +from wrolpi.common import logger from wrolpi.common import set_test_media_directory, Base, set_test_config from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts from wrolpi.dates import set_test_now @@ -47,6 +48,8 @@ from wrolpi.files.models import Directory, FileGroup from wrolpi.tags import Tag, upsert_tag from wrolpi.vars import PROJECT_DIR +logger = logger.getChild(__name__) + def get_test_db_engine(): suffix = str(uuid1()).replace('-', '') diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index ee12fab0..c0636a23 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -27,7 +27,7 @@ from wrolpi import flags from wrolpi.api_utils import api_app from wrolpi.cmd import pid_is_running from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \ - limit_concurrent, wrol_mode_enabled + limit_concurrent, wrol_mode_enabled, background_task from wrolpi.dates import TZDateTime, now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload @@ -226,7 +226,7 @@ class Download(ModelHelper, Base): # noqa from modules.videos.lib import save_channels_config save_channels_config() # Save download config again because this download is now removed from the download lists. - api_app.add_task(save_downloads_config()) + background_task(save_downloads_config()) @staticmethod def get_by_id(id_: int, session: Session = None) -> Optional[Base]: @@ -538,9 +538,9 @@ class DownloadManager: session.flush(downloads) try: # Start downloading ASAP. - api_app.add_task(self.dispatch_downloads()) + background_task(self.dispatch_downloads()) # Save the config now that new Downloads exist. - api_app.add_task(save_downloads_config()) + background_task(save_downloads_config()) except RuntimeError: # Event loop isn't running. Probably testing? if not PYTEST: @@ -574,9 +574,6 @@ class DownloadManager: from modules.videos.models import Channel if channel := Channel.get_by_url(url=download.url, session=session): download.channel_id = channel.id - if destination := (download.settings or dict()).get('destination'): - if channel := Channel.get_by_path(destination, session): - download.channel_id = channel.id session.commit() @@ -668,7 +665,7 @@ class DownloadManager: with get_db_curs(commit=True) as curs: curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'") - api_app.add_task(save_downloads_config()) + background_task(save_downloads_config()) DOWNLOAD_SORT = ( DownloadStatus.pending, @@ -742,7 +739,7 @@ class DownloadManager: session.commit() # Save the config now that some Downloads renewed. - api_app.add_task(save_downloads_config()) + background_task(save_downloads_config()) @staticmethod def get_downloads(session: Session) -> List[Download]: @@ -1025,7 +1022,7 @@ class DownloadManager: stmt = self._delete_downloads_q(once=True) deleted_ids = [i for i, in session.execute(stmt).fetchall()] session.commit() - api_app.add_task(save_downloads_config()) + background_task(save_downloads_config()) return deleted_ids @staticmethod @@ -1130,7 +1127,7 @@ async def signal_download_download(download_id: int, download_url: str): # Allow the download to resume. download_manager.unkill_download(download_id) # Save the config now that the Download has finished. - api_app.add_task(save_downloads_config()) + background_task(save_downloads_config()) except asyncio.CancelledError as e: worker_logger.warning('Canceled!', exc_info=e) return diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 09282acc..e0ffb19b 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -137,7 +137,7 @@ def get_settings(_: Request): ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories] settings = { - 'archive_directory': config.archive_directory, + 'archive_directory': config.archive_destination, 'download_manager_disabled': download_manager.is_disabled, 'download_manager_stopped': download_manager.is_stopped, 'download_on_startup': config.download_on_startup, @@ -150,15 +150,15 @@ def get_settings(_: Request): 'ignore_outdated_zims': config.ignore_outdated_zims, 'ignored_directories': ignored_directories, 'log_level': api_app.shared_ctx.log_level.value, - 'map_directory': config.map_directory, + 'map_directory': config.map_destination, 'nav_color': config.nav_color, 'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative. 'throttle_on_startup': config.throttle_on_startup, 'throttle_status': admin.throttle_status().name, 'version': __version__, - 'videos_directory': config.videos_directory, + 'videos_directory': config.videos_destination, 'wrol_mode': config.wrol_mode, - 'zims_directory': config.zims_directory, + 'zims_directory': config.zims_destination, } return json_response(settings) @@ -190,25 +190,25 @@ async def update_settings(_: Request, body: schema.SettingsRequest): if not config: raise InvalidConfig() - if body.archive_directory and pathlib.Path(body.archive_directory).is_absolute(): + if body.archive_destination and pathlib.Path(body.archive_destination).is_absolute(): raise InvalidConfig('Archive directory must be relative to media directory') - elif not body.archive_directory: - config['archive_directory'] = wrolpi_config.default_config['archive_directory'] + elif not body.archive_destination: + config['archive_destination'] = wrolpi_config.default_config['archive_destination'] - if body.videos_directory and pathlib.Path(body.videos_directory).is_absolute(): + if body.videos_destination and pathlib.Path(body.videos_destination).is_absolute(): raise InvalidConfig('Videos directory must be relative to media directory') - elif not body.videos_directory: - config['videos_directory'] = wrolpi_config.default_config['videos_directory'] + elif not body.videos_destination: + config['videos_destination'] = wrolpi_config.default_config['videos_destination'] - if body.map_directory and pathlib.Path(body.map_directory).is_absolute(): + if body.map_destination and pathlib.Path(body.map_destination).is_absolute(): raise InvalidConfig('Map directory must be relative to media directory') - elif not body.map_directory: - config['map_directory'] = wrolpi_config.default_config['map_directory'] + elif not body.map_destination: + config['map_destination'] = wrolpi_config.default_config['map_destination'] - if body.zims_directory and pathlib.Path(body.zims_directory).is_absolute(): + if body.zims_destination and pathlib.Path(body.zims_destination).is_absolute(): raise InvalidConfig('Zims directory must be relative to media directory') - elif not body.zims_directory: - config['zims_directory'] = wrolpi_config.default_config['zims_directory'] + elif not body.zims_destination: + config['zims_destination'] = wrolpi_config.default_config['zims_destination'] log_level = config.pop('log_level', None) if isinstance(log_level, int): diff --git a/wrolpi/schema.py b/wrolpi/schema.py index f7bc6aed..2fadb3c5 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -32,7 +32,7 @@ class SemanticUIColors(enum.StrEnum): @dataclass class SettingsRequest: - archive_directory: Optional[str] = None + archive_destination: Optional[str] = None download_on_startup: Optional[bool] = None download_timeout: Optional[int] = None hotspot_device: Optional[str] = None @@ -42,14 +42,14 @@ class SettingsRequest: hotspot_status: Optional[bool] = None ignore_outdated_zims: Optional[bool] = None log_level: Optional[int] = None - map_directory: Optional[str] = None + map_destination: Optional[str] = None nav_color: Optional[str] = None media_directory: Optional[str] = None throttle_on: Optional[bool] = None throttle_on_startup: Optional[bool] = None - videos_directory: Optional[str] = None + videos_destination: Optional[str] = None wrol_mode: Optional[bool] = None - zims_directory: Optional[str] = None + zims_destination: Optional[str] = None def __post_init__(self): self.nav_color = self.nav_color.lower() if self.nav_color else None diff --git a/wrolpi/tags.py b/wrolpi/tags.py index d55e324a..14bd63eb 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -426,7 +426,6 @@ def delete_tag(tag_id: int, session: Session = None): sync_tags_directory() -@register_refresh_cleanup @optional_session def import_tags_config(session: Session = None): """Reads the Tags and TagFiles from the config file, upserts them in the DB.""" diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 76c7e0d2..91c0aa82 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -530,14 +530,15 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto request, response = await test_async_client.post('/api/search_suggestions', json=body) assert response.status_code == HTTPStatus.OK if expected_channels: - assert response.json['channels'] == expected_channels + for channel, expected_channel in zip_longest(response.json['channels'], expected_channels): + assert_dict_contains(channel, expected_channel) assert response.json['domains'] == expected_domains await assert_results( dict(search_str='foo'), [ - {'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}, - {'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []}, + {'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}, + {'directory': 'videos/Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []}, ], [{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}], ) @@ -546,14 +547,14 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto await assert_results( dict(search_str='foo l'), [ - {'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}], + {'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}], [], ) await assert_results( dict(search_str='bar'), [ - {'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []} + {'directory': 'videos/Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []} ], [{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}], ) @@ -705,29 +706,28 @@ def test_recursive_errors(): @pytest.mark.asyncio async def test_settings_special_directories(test_async_client, test_config): """Maintainer can change special directories.""" - assert get_wrolpi_config().archive_directory == 'archive' - data = {'archive_directory': 'archives'} + data = {'archive_destination': 'archives'} request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) assert response.status_code == HTTPStatus.NO_CONTENT - assert get_wrolpi_config().archive_directory == 'archives' + assert get_wrolpi_config().archive_destination == 'archives' - data = {'archive_directory': '/absolute/not/allowed'} + data = {'archive_destination': '/absolute/not/allowed'} request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) assert response.status_code == HTTPStatus.BAD_REQUEST - data = {'videos_directory': '/absolute/not/allowed'} + data = {'videos_destination': '/absolute/not/allowed'} request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) assert response.status_code == HTTPStatus.BAD_REQUEST - data = {'map_directory': '/absolute/not/allowed'} + data = {'map_destination': '/absolute/not/allowed'} request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) assert response.status_code == HTTPStatus.BAD_REQUEST - data = {'zims_directory': '/absolute/not/allowed'} + data = {'zims_destination': '/absolute/not/allowed'} request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) assert response.status_code == HTTPStatus.BAD_REQUEST # Empty directory restores default. - data = {'archive_directory': ''} + data = {'archive_destination': ''} request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) assert response.status_code == HTTPStatus.NO_CONTENT - assert get_wrolpi_config().archive_directory == 'archive' + assert get_wrolpi_config().archive_destination == 'archive/%(domain)s'