mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
A Channel can be tagged with a single Tag. The Channel can be moved into a Tag Directory according to the config.
This commit is contained in:
Roland
parent
2b8ef79561
commit
45167e4f32
39 changed files with 691 additions and 165 deletions
33
alembic/versions/d4e446637f18_channel_tag.py
Normal file
33
alembic/versions/d4e446637f18_channel_tag.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
"""Channel tag
|
||||
|
||||
Revision ID: d4e446637f18
|
||||
Revises: 2bfb3b50b178
|
||||
Create Date: 2024-08-03 17:49:17.133086
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'd4e446637f18'
|
||||
down_revision = '2bfb3b50b178'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('ALTER TABLE channel ADD COLUMN tag_id INTEGER REFERENCES tag(id)')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS tag_id')
|
||||
|
|
@ -131,8 +131,8 @@ export function useTags() {
|
|||
</Label.Group>
|
||||
}
|
||||
|
||||
const SingleTag = ({name}) => {
|
||||
return <Label.Group tag><NameToTagLabel name={name}/></Label.Group>
|
||||
const SingleTag = ({name, ...props}) => {
|
||||
return <Label.Group tag {...props}><NameToTagLabel name={name}/></Label.Group>
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
|
|
|
|||
|
|
@ -112,6 +112,32 @@ export async function getChannel(id) {
|
|||
return (await response.json())['channel'];
|
||||
}
|
||||
|
||||
export async function tagChannel(channelId, tagName, directory) {
|
||||
const body = {
|
||||
tag_name: tagName,
|
||||
}
|
||||
if (directory) {
|
||||
body['directory'] = directory;
|
||||
}
|
||||
const response = await apiPost(`${VIDEOS_API}/channels/${channelId}/tag`, body);
|
||||
if (!response.ok) {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Unexpected server response',
|
||||
description: 'Failed to tag channel. See server logs.',
|
||||
time: 5000,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function tagChannelInfo(channelId, tagName) {
|
||||
const body = {tag_name: tagName}
|
||||
const response = await apiPost(`${VIDEOS_API}/channels/${channelId}/tag_info`, body);
|
||||
if (response.ok) {
|
||||
return (await response.json()).videos_destination;
|
||||
}
|
||||
}
|
||||
|
||||
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline) {
|
||||
// Build a search query to retrieve a list of videos from the API
|
||||
offset = parseInt(offset || 0);
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import React, {useState} from "react";
|
||||
import {Dimmer, Dropdown, Grid, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react";
|
||||
import {Dimmer, Dropdown, Grid, Label, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react";
|
||||
import {
|
||||
createChannel,
|
||||
createChannelDownload,
|
||||
deleteChannel,
|
||||
deleteDownload,
|
||||
refreshChannel,
|
||||
tagChannel,
|
||||
tagChannelInfo,
|
||||
updateChannel,
|
||||
updateChannelDownload
|
||||
} from "../api";
|
||||
|
|
@ -39,7 +41,9 @@ import {
|
|||
Header,
|
||||
Loader,
|
||||
Modal,
|
||||
ModalActions,
|
||||
ModalContent,
|
||||
ModalHeader,
|
||||
Segment,
|
||||
Statistic
|
||||
} from "./Theme";
|
||||
|
|
@ -47,7 +51,7 @@ import {Media, ThemeContext} from "../contexts/contexts";
|
|||
import {SortableTable} from "./SortableTable";
|
||||
import {toast} from "react-semantic-toasts-2";
|
||||
import {RecurringDownloadsTable} from "./admin/Downloads";
|
||||
import {TagsSelector} from "../Tags";
|
||||
import {TagsContext, TagsSelector} from "../Tags";
|
||||
|
||||
|
||||
function ChannelStatistics({statistics}) {
|
||||
|
|
@ -85,22 +89,31 @@ function ChannelPage({create, header}) {
|
|||
const [messageHeader, setMessageHeader] = useState();
|
||||
const [messageContent, setMessageContent] = useState();
|
||||
const [downloadModalOpen, setDownloadModalOpen] = useState(false);
|
||||
const [rssDownloadOpen, setRssDownloadOpen] = useState(false);
|
||||
|
||||
const navigate = useNavigate();
|
||||
const {channelId} = useParams();
|
||||
const {channel, changeValue, fetchChannel} = useChannel(channelId);
|
||||
const {SingleTag} = React.useContext(TagsContext);
|
||||
|
||||
let title;
|
||||
if (channel && channel.name) {
|
||||
title = channel.name;
|
||||
}
|
||||
useTitle(title);
|
||||
const [tagModalOpen, setTagModalOpen] = useState(false);
|
||||
const [newTagName, setNewTagName] = useState(channel ? channel.tag_name : null);
|
||||
const [moveToTagDirectory, setMoveToTagDirectory] = useState(true);
|
||||
const [newTagDirectory, setNewTagDirectory] = useState('');
|
||||
|
||||
if (!channel) {
|
||||
useTitle(_.isEmpty(channel) ? null : `${channel.name} Channel`);
|
||||
|
||||
React.useEffect(() => {
|
||||
console.debug(channel);
|
||||
if (!_.isEmpty(channel)) {
|
||||
setNewTagName(channel.tag_name);
|
||||
}
|
||||
}, [JSON.stringify(channel)]);
|
||||
|
||||
if (_.isEmpty(channel)) {
|
||||
return <Loader active/>;
|
||||
}
|
||||
|
||||
|
||||
const handleCheckbox = (e, {name, checked}) => {
|
||||
if (e) {
|
||||
e.preventDefault();
|
||||
|
|
@ -247,6 +260,64 @@ function ChannelPage({create, header}) {
|
|||
setDownloadModalOpen(false);
|
||||
}
|
||||
|
||||
const handleTagModalOpen = (e) => {
|
||||
if (e) {
|
||||
e.preventDefault();
|
||||
}
|
||||
setTagModalOpen(true);
|
||||
}
|
||||
|
||||
const handleTagChannel = async () => {
|
||||
try {
|
||||
await tagChannel(channelId, newTagName, moveToTagDirectory ? newTagDirectory : null);
|
||||
setTagModalOpen(false);
|
||||
} catch (e) {
|
||||
console.error('Failed to tag channel', e);
|
||||
} finally {
|
||||
await fetchChannel();
|
||||
}
|
||||
}
|
||||
|
||||
const handleTagSelect = async (newTagName_) => {
|
||||
setNewTagName(newTagName_);
|
||||
try {
|
||||
const videosDestination = await tagChannelInfo(channelId, newTagName_);
|
||||
setNewTagDirectory(videosDestination);
|
||||
} catch (e) {
|
||||
console.error('Failed to tag channel', e);
|
||||
}
|
||||
}
|
||||
|
||||
const tagModal = <Modal
|
||||
open={tagModalOpen}
|
||||
onClose={() => setTagModalOpen(false)}
|
||||
closeIcon
|
||||
>
|
||||
<ModalHeader>{channel.tag_name ? 'Modify Tag' : 'Add Tag'}</ModalHeader>
|
||||
<ModalContent>
|
||||
<TagsSelector
|
||||
limit={1}
|
||||
selectedTagNames={newTagName ? [newTagName] : []}
|
||||
onAdd={handleTagSelect}
|
||||
onRemove={() => handleTagSelect(null)}
|
||||
/>
|
||||
<br/>
|
||||
<Toggle
|
||||
label='Move to directory: '
|
||||
checked={moveToTagDirectory}
|
||||
onChange={setMoveToTagDirectory}
|
||||
/>
|
||||
{newTagDirectory && <Label size='large'>{newTagDirectory}</Label>}
|
||||
</ModalContent>
|
||||
<ModalActions>
|
||||
<Button onClick={() => setTagModalOpen(false)}>Cancel</Button>
|
||||
<Button
|
||||
color='violet'
|
||||
onClick={handleTagChannel}
|
||||
>Save</Button>
|
||||
</ModalActions>
|
||||
</Modal>;
|
||||
|
||||
return <>
|
||||
<BackButton/>
|
||||
{!create &&
|
||||
|
|
@ -324,6 +395,8 @@ function ChannelPage({create, header}) {
|
|||
</FormField>
|
||||
</FormGroup>
|
||||
|
||||
{channel.tag_name && <SingleTag name={channel.tag_name}/>}
|
||||
|
||||
<Message error
|
||||
header={messageHeader}
|
||||
content={messageContent}
|
||||
|
|
@ -356,6 +429,12 @@ function ChannelPage({create, header}) {
|
|||
>Refresh</APIButton>
|
||||
</>
|
||||
}
|
||||
<Button
|
||||
size='small'
|
||||
onClick={handleTagModalOpen}
|
||||
color='green'
|
||||
>Tag</Button>
|
||||
{tagModal}
|
||||
</Grid.Column>
|
||||
<Grid.Column>
|
||||
<APIButton
|
||||
|
|
@ -587,6 +666,8 @@ export function ChannelNewPage(props) {
|
|||
}
|
||||
|
||||
function ChannelRow({channel}) {
|
||||
const {SingleTag} = React.useContext(TagsContext);
|
||||
|
||||
const videosTo = `/videos/channel/${channel.id}/video`;
|
||||
|
||||
const {inverted} = React.useContext(ThemeContext);
|
||||
|
|
@ -597,6 +678,9 @@ function ChannelRow({channel}) {
|
|||
<TableCell>
|
||||
<Link to={videosTo}>{channel.name}</Link>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{channel.tag_name ? <SingleTag name={channel.tag_name}/> : null}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{channel.video_count}
|
||||
</TableCell>
|
||||
|
|
@ -613,6 +697,8 @@ function ChannelRow({channel}) {
|
|||
}
|
||||
|
||||
function MobileChannelRow({channel}) {
|
||||
const {SingleTag} = React.useContext(TagsContext);
|
||||
|
||||
const editTo = `/videos/channel/${channel.id}/edit`;
|
||||
const videosTo = `/videos/channel/${channel.id}/video`;
|
||||
return <TableRow verticalAlign='top'>
|
||||
|
|
@ -621,6 +707,7 @@ function MobileChannelRow({channel}) {
|
|||
<h3>
|
||||
{channel.name}
|
||||
</h3>
|
||||
{channel.tag_name ? <SingleTag name={channel.tag_name}/> : null}
|
||||
</Link>
|
||||
<p>
|
||||
Videos: {channel.video_count}
|
||||
|
|
@ -669,6 +756,7 @@ export function ChannelsPage() {
|
|||
|
||||
const headers = [
|
||||
{key: 'name', text: 'Name', sortBy: 'name', width: 8},
|
||||
{key: 'tag', text: 'Tag', sortBy: 'tag', width: 2},
|
||||
{key: 'video_count', text: 'Videos', sortBy: 'video_count', width: 2},
|
||||
{key: 'download_frequency', text: 'Download Frequency', sortBy: 'minimum_frequency', width: 2},
|
||||
{key: 'size', text: 'Size', sortBy: 'size', width: 2},
|
||||
|
|
|
|||
|
|
@ -734,6 +734,10 @@ export const useDirectories = (defaultDirectory) => {
|
|||
fetchDirectories();
|
||||
}, [directory, defaultDirectory]);
|
||||
|
||||
useEffect(() => {
|
||||
setDirectory(defaultDirectory);
|
||||
}, [defaultDirectory])
|
||||
|
||||
return {directory, directories, setDirectory, exists, isDir, isFile};
|
||||
}
|
||||
|
||||
|
|
|
|||
4
main.py
4
main.py
|
|
@ -322,9 +322,9 @@ async def perpetual_have_internet_worker():
|
|||
# Check hourly once we have internet.
|
||||
await asyncio.sleep(float(Seconds.hour))
|
||||
else:
|
||||
# Check every minute until the internet is back.
|
||||
# Check more often until the internet is back.
|
||||
flags.have_internet.clear()
|
||||
await asyncio.sleep(float(Seconds.minute))
|
||||
await asyncio.sleep(10)
|
||||
except Exception as e:
|
||||
logger.error('Failed to check if internet is up', exc_info=e)
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,10 @@ class ArchiveFiles:
|
|||
|
||||
|
||||
def get_archive_directory() -> pathlib.Path:
|
||||
archive_directory = get_media_directory() / get_wrolpi_config().archive_directory
|
||||
archive_destination = get_wrolpi_config().archive_destination
|
||||
variables = dict(domain='', year='', month='', day='')
|
||||
archive_destination = archive_destination % variables
|
||||
archive_directory = get_media_directory() / archive_destination
|
||||
return archive_directory
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ from sqlalchemy.orm import relationship, Session, validates
|
|||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.common import ModelHelper, Base, logger, get_title_from_html
|
||||
from wrolpi.common import ModelHelper, Base, logger, get_title_from_html, get_wrolpi_config, get_media_directory
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import optional_session
|
||||
from wrolpi.errors import UnknownArchive
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -342,3 +343,21 @@ class Domain(Base, ModelHelper):
|
|||
if len(value.split('.')) < 2:
|
||||
raise ValueError(f'Domain must contain at least one "." domain={repr(value)}')
|
||||
return value
|
||||
|
||||
@property
|
||||
def download_directory(self) -> pathlib.Path:
|
||||
archive_destination = get_wrolpi_config().archive_destination
|
||||
|
||||
now_ = now()
|
||||
variables = dict(
|
||||
domain=self.domain,
|
||||
year=now_.year,
|
||||
month=now_.month,
|
||||
day=now_.day,
|
||||
)
|
||||
archive_destination = archive_destination % variables
|
||||
archive_destination = pathlib.Path(archive_destination.lstrip('/'))
|
||||
if not archive_destination.is_absolute():
|
||||
archive_destination = get_media_directory() / archive_destination
|
||||
|
||||
return archive_destination
|
||||
|
|
|
|||
|
|
@ -820,12 +820,13 @@ def test_get_url_from_singlefile():
|
|||
assert lib.get_url_from_singlefile(SINGLEFILE_EXAMPLE_1) == 'https://www.example.com'
|
||||
|
||||
|
||||
def test_get_custom_archive_directory(test_directory, test_config):
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_custom_archive_directory(test_async_client, test_directory, test_config):
|
||||
"""Custom directory can be used for archive directory."""
|
||||
# Default location.
|
||||
assert lib.get_archive_directory() == (test_directory / 'archive')
|
||||
|
||||
get_wrolpi_config().archive_directory = 'custom/archives'
|
||||
get_wrolpi_config().archive_destination = 'custom/archives'
|
||||
|
||||
assert lib.get_archive_directory() == (test_directory / 'custom/archives')
|
||||
assert lib.get_domain_directory('https://example.com') == (test_directory / 'custom/archives/example.com')
|
||||
|
|
|
|||
27
modules/archive/test/test_models.py
Normal file
27
modules/archive/test/test_models.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.archive import Domain
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_download_destination(test_async_client, test_session, test_directory, archive_factory, fake_now):
|
||||
fake_now(datetime(2000, 1, 2))
|
||||
|
||||
wrolpi_config = get_wrolpi_config()
|
||||
|
||||
archive_factory(domain='wrolpi.org')
|
||||
domain = test_session.query(Domain).one()
|
||||
|
||||
# Test the default download directory.
|
||||
assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org')
|
||||
|
||||
# Year of download is supported.
|
||||
wrolpi_config.archive_destination = 'archives/%(domain)s/%(year)s'
|
||||
assert str(domain.download_directory) == str(test_directory / 'archives/wrolpi.org/2000')
|
||||
|
||||
# More download date is supported.
|
||||
wrolpi_config.archive_destination = 'archive/%(domain)s/%(year)s/%(month)s/%(day)s'
|
||||
assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org/2000/1/2')
|
||||
|
|
@ -19,7 +19,7 @@ logger = logger.getChild(__name__)
|
|||
|
||||
|
||||
def get_map_directory() -> Path:
|
||||
map_directory = get_media_directory() / get_wrolpi_config().map_directory
|
||||
map_directory = get_media_directory() / get_wrolpi_config().map_destination
|
||||
if not map_directory.is_dir():
|
||||
map_directory.mkdir(parents=True)
|
||||
return map_directory
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ def test_get_custom_map_directory(test_directory, test_config):
|
|||
# Default location.
|
||||
assert lib.get_map_directory() == (test_directory / 'map')
|
||||
|
||||
get_wrolpi_config().map_directory = 'custom/deep/map/directory'
|
||||
get_wrolpi_config().map_destination = 'custom/deep/map/directory'
|
||||
|
||||
assert lib.get_map_directory() == (test_directory / 'custom/deep/map/directory')
|
||||
assert (test_directory / 'custom/deep/map/directory').is_dir()
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from wrolpi.vars import PYTEST
|
|||
from . import lib
|
||||
from .. import schema
|
||||
from ..errors import UnknownChannel
|
||||
from ..lib import save_channels_config
|
||||
from ..lib import save_channels_config, format_videos_destination
|
||||
from ..models import Channel
|
||||
|
||||
channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels')
|
||||
|
|
@ -136,3 +136,41 @@ async def channel_download(_: Request, channel_id: int, body: schema.ChannelDown
|
|||
# Warn the user that downloads are disabled.
|
||||
Events.send_downloads_disabled('Channel download created. But downloads are disabled.')
|
||||
return response.empty()
|
||||
|
||||
|
||||
@channel_bp.post('/<channel_id:int>/tag_info')
|
||||
@openapi.definition(
|
||||
description='Get Channel Tag Info',
|
||||
body=schema.ChannelTagInfoRequest,
|
||||
)
|
||||
@validate(schema.ChannelTagInfoRequest)
|
||||
@run_after(save_channels_config)
|
||||
@wrol_mode_check
|
||||
async def channel_tag_info(_: Request, channel_id: int, body: schema.ChannelTagInfoRequest):
|
||||
from wrolpi.tags import Tag
|
||||
channel_name = Channel.find_by_id(channel_id).name
|
||||
tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None
|
||||
videos_destination = format_videos_destination(channel_name, tag_name)
|
||||
ret = dict(videos_destination=videos_destination)
|
||||
return json_response(ret)
|
||||
|
||||
|
||||
@channel_bp.post('/<channel_id:int>/tag')
|
||||
@openapi.definition(
|
||||
description='Tag Channel',
|
||||
body=schema.ChannelTagRequest,
|
||||
)
|
||||
@openapi.response(HTTPStatus.NO_CONTENT)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
@validate(schema.ChannelTagRequest)
|
||||
@run_after(save_channels_config)
|
||||
@wrol_mode_check
|
||||
async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagRequest):
|
||||
directory = None
|
||||
if body.directory:
|
||||
directory = pathlib.Path(body.directory)
|
||||
directory = get_media_directory() / directory
|
||||
|
||||
await lib.tag_channel(body.tag_name, directory, channel_id)
|
||||
|
||||
return response.raw('', HTTPStatus.NO_CONTENT)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import pathlib
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Union
|
||||
|
||||
from sqlalchemy import or_, func, desc, asc
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from wrolpi.common import run_after, logger, \
|
||||
get_media_directory
|
||||
|
|
@ -12,7 +12,7 @@ from wrolpi.downloader import save_downloads_config, download_manager
|
|||
from wrolpi.errors import UnknownDirectory, APIError, ValidationError
|
||||
from .. import schema
|
||||
from ..common import check_for_channel_conflicts
|
||||
from ..errors import UnknownChannel
|
||||
from ..errors import UnknownChannel, ChannelDirectoryConflict
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Channel, Video
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ async def get_minimal_channels() -> List[dict]:
|
|||
# one that will consume the most resources.
|
||||
stmt = '''
|
||||
SELECT
|
||||
c.id, name, directory, c.url,
|
||||
c.id, c.name AS "name", directory, c.url, t.name AS "tag_name",
|
||||
COUNT(v.id) as video_count,
|
||||
SUM(fg.size)::BIGINT AS size,
|
||||
(
|
||||
|
|
@ -40,7 +40,8 @@ async def get_minimal_channels() -> List[dict]:
|
|||
channel AS c
|
||||
LEFT JOIN video v on c.id = v.channel_id
|
||||
LEFT JOIN file_group fg on fg.id = v.file_group_id
|
||||
GROUP BY 1, 2, 3, 4
|
||||
LEFT JOIN tag t ON t.id = c.tag_id
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
channels = sorted([dict(i) for i in curs.fetchall()], key=lambda i: i['name'].lower())
|
||||
|
|
@ -115,10 +116,7 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No
|
|||
@optional_session
|
||||
def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
|
||||
"""Update a Channel's DB record"""
|
||||
try:
|
||||
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
|
||||
except NoResultFound:
|
||||
raise UnknownChannel()
|
||||
channel = Channel.find_by_id(channel_id, session)
|
||||
|
||||
# Only update directory if it was empty
|
||||
if data.directory and not channel.directory:
|
||||
|
|
@ -237,8 +235,9 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
|
|||
Channel.find_by_id(channel_id, session=session)
|
||||
download = download_manager.get_download(session, id_=download_id)
|
||||
download.url = url
|
||||
download.frequency = download.frequency or frequency
|
||||
download.frequency = frequency
|
||||
download.settings = settings
|
||||
session.commit()
|
||||
|
||||
download_manager.remove_from_skip_list(url)
|
||||
|
||||
|
|
@ -246,3 +245,38 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
|
|||
await save_downloads_config()
|
||||
|
||||
return download
|
||||
|
||||
|
||||
@optional_session
|
||||
async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id: int, session: Session = None):
|
||||
"""Add a Tag to a Channel, or remove a Tag from a Channel if no `tag_name` is provided.
|
||||
|
||||
Move the Channel to the new directory, if provided."""
|
||||
from wrolpi.tags import Tag
|
||||
|
||||
channel = Channel.find_by_id(channel_id, session=session)
|
||||
|
||||
if tag_name:
|
||||
tag = Tag.find_by_name(tag_name, session=session)
|
||||
channel.tag_id = tag.id
|
||||
channel.tag = tag
|
||||
else:
|
||||
channel.tag = channel.tag_id = None
|
||||
|
||||
session.flush([channel, ])
|
||||
|
||||
# Do not move Channel files into a directory with files.
|
||||
if directory and directory.is_dir() and next(directory.iterdir(), None):
|
||||
raise ChannelDirectoryConflict('Channel directory already exists and is not empty')
|
||||
|
||||
# Only move Channel when requested.
|
||||
if directory:
|
||||
# Move to newly defined directory only if necessary.
|
||||
logger.info(f'Tagging {channel} with {tag_name} moving to {directory} from {channel.directory}')
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
if channel.directory != directory:
|
||||
await channel.move(directory, session)
|
||||
else:
|
||||
logger.info(f'Tagging {channel} with {tag_name}')
|
||||
|
||||
session.commit()
|
||||
|
|
|
|||
|
|
@ -8,8 +8,11 @@ from http import HTTPStatus
|
|||
import mock
|
||||
import pytest
|
||||
|
||||
from modules.videos.common import get_videos_directory
|
||||
from modules.videos.downloader import ChannelDownloader
|
||||
from modules.videos.models import Channel
|
||||
from modules.videos.lib import save_channels_config, get_channels_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.common import get_relative_to_media_directory, walk
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import download_manager, Download, DownloadResult
|
||||
|
|
@ -412,7 +415,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_channel_download(test_async_client, test_session, simple_channel, tag_factory):
|
||||
async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory):
|
||||
tag = tag_factory()
|
||||
|
||||
# Create Download.
|
||||
|
|
@ -452,3 +455,93 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
|
|||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert test_session.query(Download).count() == 1
|
||||
assert test_session.query(Channel).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory, video_factory,
|
||||
test_channels_config, test_download_manager, test_downloader):
|
||||
"""A single Tag can be applied to a Channel."""
|
||||
# Create channel directory in the usual videos directory.
|
||||
videos_directory = get_videos_directory()
|
||||
channel = channel_factory(name='Channel Name', download_frequency=120)
|
||||
channel_directory = channel.directory
|
||||
tag = tag_factory('Tag Name')
|
||||
v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
|
||||
# Create recurring download which uses the Channel's directory.
|
||||
download = download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
|
||||
settings={
|
||||
'destination': str(test_directory / 'videos/Channel Name')})
|
||||
test_session.commit()
|
||||
save_channels_config()
|
||||
assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Channel Name')
|
||||
# Channel download downloads into the Channel's directory.
|
||||
assert channel.downloads[0].settings['destination'] == str(test_directory / 'videos/Channel Name')
|
||||
# Make extra file in the Channel's directory, it should be moved.
|
||||
(channel_directory / 'extra file.txt').write_text('extra file contents')
|
||||
# Channel directory is in the Videos directory.
|
||||
assert channel.directory == channel_directory
|
||||
# Videos are in the Channel's Directory.
|
||||
assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4'
|
||||
assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4'
|
||||
video_files = [i for i in walk(videos_directory) if i.is_file()]
|
||||
assert len(video_files) == 3
|
||||
|
||||
body = dict(tag_name=tag.name, directory='videos/Tag Name/Channel Name')
|
||||
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
channel = test_session.query(Channel).one()
|
||||
assert channel.tag_name == tag.name
|
||||
# Channel was moved to Tag's directory, old directory was removed
|
||||
new_channel_directory = test_directory / 'videos/Tag Name/Channel Name'
|
||||
assert channel.directory == new_channel_directory, 'Channel directory should have been changed.'
|
||||
assert new_channel_directory.is_dir(), 'Channel should have been moved.'
|
||||
assert next(new_channel_directory.iterdir(), None), 'Channel videos should have been moved.'
|
||||
assert not channel_directory.exists(), 'Old Channel directory should have been deleted.'
|
||||
# Channel download goes into the Channel's directory.
|
||||
d1, d2 = test_session.query(Download).all()
|
||||
assert d1.settings['destination'] == str(test_directory / 'videos/Tag Name/Channel Name'), \
|
||||
f'{d1} was not moved'
|
||||
assert d2.settings['destination'] == str(test_directory / 'videos/Tag Name/Channel Name'), \
|
||||
f'{d2} was not moved'
|
||||
|
||||
# Videos were moved
|
||||
v1, v2 = test_session.query(Video).order_by(Video.id).all()
|
||||
assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Tag Name/Channel Name/video1.mp4'
|
||||
assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Tag Name/Channel Name/video2.mp4'
|
||||
# Extra file was also moved.
|
||||
assert (new_channel_directory / 'extra file.txt').read_text() == 'extra file contents'
|
||||
# No new files were created.
|
||||
assert len([i for i in walk(videos_directory) if i.is_file()]) == 3
|
||||
|
||||
assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Tag Name/Channel Name')
|
||||
|
||||
# Remove tag, Channel/Videos should be moved back.
|
||||
body = dict(tag_name=None, directory='videos/Channel Name')
|
||||
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
|
||||
channel = test_session.query(Channel).one()
|
||||
assert channel.tag_name is None
|
||||
assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
|
||||
v1, v2 = test_session.query(Video).order_by(Video.id).all()
|
||||
assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4'
|
||||
assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4'
|
||||
assert not (test_directory / 'videos/Tag Name/Channel Name').exists()
|
||||
# Downloads are moved back.
|
||||
d1, d2 = test_session.query(Download).all()
|
||||
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
|
||||
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
|
||||
|
||||
# Channel can be Tagged, but not move directories.
|
||||
body = dict(tag_name=tag.name)
|
||||
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
|
||||
channel = test_session.query(Channel).one()
|
||||
assert channel.tag_name == 'Tag Name'
|
||||
assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
|
||||
assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4'
|
||||
assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4'
|
||||
# Downloads were not changed.
|
||||
d1, d2 = test_session.query(Download).all()
|
||||
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
|
||||
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
|
||||
|
|
|
|||
|
|
@ -3,10 +3,9 @@ from http import HTTPStatus
|
|||
|
||||
import pytest
|
||||
|
||||
from modules.videos.downloader import VideoDownloader
|
||||
from modules.videos.models import Channel
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.downloader import Download, DownloadFrequency, download_manager, RSSDownloader
|
||||
from wrolpi.downloader import Download, DownloadFrequency
|
||||
|
||||
|
||||
def test_delete_channel_no_url(test_session, test_client, channel_factory):
|
||||
|
|
@ -102,17 +101,3 @@ async def test_channel_download_relationships(test_session, download_channel):
|
|||
# Delete the Channel, and any relationships.
|
||||
download_channel.delete_with_videos()
|
||||
assert test_session.query(Download).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_create_channel_download(test_session, simple_channel):
|
||||
"""A Download will be related to a Channel if a Download is created in the Channel's directory"""
|
||||
test_session.flush()
|
||||
|
||||
settings = dict(destination=str(simple_channel.directory))
|
||||
download_manager.recurring_download('https://example.com/2', 1, RSSDownloader.name, test_session,
|
||||
sub_downloader_name=VideoDownloader.name, settings=settings)
|
||||
download = test_session.query(Download).one()
|
||||
assert download.settings['destination'] == str(simple_channel.directory)
|
||||
download = test_session.query(Download).one()
|
||||
assert download.channel == simple_channel
|
||||
|
|
|
|||
|
|
@ -35,8 +35,9 @@ class ConfigError(Exception):
|
|||
|
||||
def get_videos_directory() -> pathlib.Path:
|
||||
"""Get the "videos" directory in the media directory. Make it if it does not exist."""
|
||||
config = get_wrolpi_config()
|
||||
directory = get_media_directory() / config.videos_directory
|
||||
from modules.videos.lib import format_videos_destination
|
||||
videos_destination = format_videos_destination()
|
||||
directory = get_media_directory() / videos_destination
|
||||
if not directory.is_dir():
|
||||
directory.mkdir(parents=True)
|
||||
return directory
|
||||
|
|
|
|||
|
|
@ -11,12 +11,11 @@ import pytest
|
|||
from PIL import Image
|
||||
|
||||
from modules.videos.downloader import VideoDownloader, ChannelDownloader
|
||||
from modules.videos.lib import set_test_channels_config, set_test_downloader_config
|
||||
from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.downloader import DownloadFrequency, DownloadManager
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import Tag
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
|
|
@ -36,25 +35,25 @@ def simple_channel(test_session, test_directory) -> Channel:
|
|||
@pytest.fixture
|
||||
def channel_factory(test_session, test_directory):
|
||||
"""Create a random Channel with a directory, and Download."""
|
||||
from wrolpi.tags import Tag
|
||||
|
||||
def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None,
|
||||
directory: pathlib.Path = None):
|
||||
directory: pathlib.Path = None, tag_name: str = None):
|
||||
name = name or str(uuid4())
|
||||
directory = directory or test_directory / name
|
||||
if not directory.is_dir():
|
||||
directory.mkdir()
|
||||
tag = Tag.get_by_name(tag_name) if tag_name else None
|
||||
channel = Channel(
|
||||
directory=directory, # noqa
|
||||
name=name,
|
||||
url=url or f'https://example.com/{name}',
|
||||
source_id=source_id,
|
||||
tag=tag,
|
||||
)
|
||||
channel.directory = directory or format_videos_destination(name, tag, channel.url)
|
||||
channel.directory.mkdir(exist_ok=True, parents=True)
|
||||
test_session.add(channel)
|
||||
test_session.flush([channel])
|
||||
assert channel.id and channel.url
|
||||
if download_frequency:
|
||||
download = channel.get_or_create_download(channel.url, 60)
|
||||
download.frequency = download_frequency
|
||||
download = channel.get_or_create_download(channel.url, download_frequency)
|
||||
assert download.url == channel.url
|
||||
assert channel.downloads
|
||||
test_session.commit()
|
||||
|
|
@ -106,8 +105,9 @@ def video_factory(test_session, test_directory):
|
|||
path = pathlib.Path(with_video_file) if not isinstance(with_video_file, pathlib.Path) else with_video_file
|
||||
elif channel_id:
|
||||
# Put the video in its Channel's directory.
|
||||
channel = test_session.query(Channel).filter_by(id=channel_id).one()
|
||||
channel = Channel.find_by_id(channel_id)
|
||||
path = (channel.directory or test_directory) / f'{title}.mp4'
|
||||
path.parent.mkdir(exist_ok=True, parents=True)
|
||||
else:
|
||||
# Put any video not in a Channel and without a specified file in the NO CHANNEL directory.
|
||||
(test_directory / 'videos/NO CHANNEL').mkdir(exist_ok=True, parents=True)
|
||||
|
|
|
|||
|
|
@ -264,7 +264,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
|
||||
channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
|
||||
channel = None
|
||||
if channel_name or channel_source_id:
|
||||
if channel_name or channel_source_id or channel_url:
|
||||
# Try to find the channel via info_json from yt-dlp.
|
||||
try:
|
||||
channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
|
||||
|
|
@ -278,7 +278,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
settings = download.settings or dict()
|
||||
destination = settings.get('destination')
|
||||
if not channel and destination:
|
||||
# Destination may override the real Channel.
|
||||
# Destination may find Channel if not already found.
|
||||
try:
|
||||
channel = get_channel(directory=destination, return_dict=False)
|
||||
found_channel = 'download_settings_directory'
|
||||
|
|
@ -290,7 +290,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
local_channel_id = settings.get('channel_id')
|
||||
channel_url = settings.get('channel_url')
|
||||
if not channel and (local_channel_id or channel_url):
|
||||
# Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
|
||||
# Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
|
||||
logger.info(f'Using download.settings to find channel')
|
||||
try:
|
||||
channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False)
|
||||
|
|
@ -320,11 +320,12 @@ class VideoDownloader(Downloader, ABC):
|
|||
logger.debug(f'Downloading {url} to destination from settings')
|
||||
elif channel:
|
||||
out_dir = channel_directory
|
||||
logger.debug(f'Downloading {url} to channel directory')
|
||||
logger.debug(f'Downloading {url} to channel directory: {channel_directory}')
|
||||
else:
|
||||
# Download to the default directory if this video has no channel.
|
||||
out_dir = get_no_channel_directory()
|
||||
logger.debug(f'Downloading {url} to default directory')
|
||||
|
||||
out_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
logs = None # noqa
|
||||
|
|
|
|||
|
|
@ -15,6 +15,12 @@ class UnknownChannel(APIError):
|
|||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class InvalidChannel(APIError):
|
||||
code = 'INVALID_CHANNEL'
|
||||
message = 'The channel is not supported'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class ChannelNameConflict(APIError):
|
||||
code = 'CHANNEL_NAME_CONFLICT'
|
||||
message = 'The channel name is already taken.'
|
||||
|
|
|
|||
|
|
@ -3,27 +3,30 @@ import dataclasses
|
|||
import html
|
||||
import pathlib
|
||||
import re
|
||||
from typing import Tuple, Optional, Generator
|
||||
from typing import Generator
|
||||
from typing import Tuple, Optional
|
||||
|
||||
import pytz
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp import YoutubeDL
|
||||
|
||||
from modules.videos.models import Video
|
||||
from wrolpi import before_startup, dates, flags
|
||||
from wrolpi.captions import extract_captions
|
||||
from wrolpi.common import ConfigFile, get_media_directory, register_refresh_cleanup, limit_concurrent
|
||||
from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \
|
||||
escape_file_name, get_media_directory
|
||||
from wrolpi.dates import Seconds, from_timestamp
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.errors import UnknownDirectory
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
|
||||
from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
|
||||
from .common import is_valid_poster, convert_image, \
|
||||
generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
|
||||
get_video_duration
|
||||
from .errors import UnknownChannel
|
||||
from .models import Channel, Video
|
||||
from .models import Channel
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -377,7 +380,6 @@ channel_import_logger = logger.getChild('channel_import')
|
|||
|
||||
|
||||
@before_startup
|
||||
@register_refresh_cleanup
|
||||
@limit_concurrent(1)
|
||||
def import_channels_config():
|
||||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
|
|
@ -645,3 +647,41 @@ def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path
|
|||
''')
|
||||
results = (pathlib.Path(j['path']) for i in curs.fetchall() for j in i[0])
|
||||
yield from results
|
||||
|
||||
|
||||
def format_videos_destination(channel_name: str = None, channel_tag: str = None, channel_url: str = None) \
|
||||
-> pathlib.Path:
|
||||
"""Return the directory where Videos should be downloaded according to the WROLPi Config.
|
||||
|
||||
@warning: Directory may or may not exist."""
|
||||
videos_destination = get_wrolpi_config().videos_destination
|
||||
|
||||
channel_domain = ''
|
||||
if channel_url:
|
||||
try:
|
||||
channel_domain = extract_domain(channel_url)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract domain from Channel URL: {channel_url}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
if channel_name and not isinstance(channel_name, str):
|
||||
raise RuntimeError('channel name must be string')
|
||||
if channel_tag and not isinstance(channel_tag, str):
|
||||
raise RuntimeError('channel tag must be string')
|
||||
|
||||
name = escape_file_name(channel_name) if channel_name else ''
|
||||
variables = dict(
|
||||
channel_name=name,
|
||||
channel_tag=channel_tag or '',
|
||||
channel_domain=channel_domain,
|
||||
)
|
||||
|
||||
try:
|
||||
videos_destination = videos_destination % variables
|
||||
except KeyError as e:
|
||||
raise FileNotFoundError(f'Cannot download to the defined "videos_destination"') from e
|
||||
|
||||
videos_destination = get_media_directory() / videos_destination.lstrip('/')
|
||||
|
||||
return videos_destination
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import json
|
||||
import pathlib
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger
|
||||
|
|
@ -11,12 +10,12 @@ from modules.videos.errors import UnknownVideo, UnknownChannel
|
|||
from wrolpi.captions import read_captions
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
from wrolpi.downloader import Download, download_manager, save_downloads_config
|
||||
from wrolpi.errors import InvalidDownload
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, move
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.tags import Tag, TagFile
|
||||
from wrolpi.tags import Tag, TagFile, save_tags_config
|
||||
from wrolpi.vars import PYTEST, DELETED_VIDEO_KEYS
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -533,10 +532,16 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
|
||||
downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id')
|
||||
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'))
|
||||
tag = relationship('Tag')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Channel id={self.id} name={repr(self.name)} url={self.url} directory={self.directory}>'
|
||||
|
||||
@property
|
||||
def tag_name(self) -> str | None:
|
||||
return self.tag.name if self.tag else None
|
||||
|
||||
def delete_with_videos(self):
|
||||
"""Delete all Video records (but not video files) related to this Channel. Then delete the Channel."""
|
||||
session = Session.object_session(self)
|
||||
|
|
@ -560,12 +565,20 @@ class Channel(ModelHelper, Base):
|
|||
channel_id = self.id
|
||||
data = data.copy()
|
||||
|
||||
# Remove unused values.
|
||||
data.pop('move_to_tag_directory', None)
|
||||
|
||||
# URL should not be empty string.
|
||||
if 'url' in data:
|
||||
data['url'] = data['url'] or None
|
||||
|
||||
session: Session = Session.object_session(self)
|
||||
|
||||
downloads = data.pop('downloads', [])
|
||||
|
||||
if tag_name := data.pop('tag_name', None):
|
||||
self.tag = Tag.find_by_name(tag_name, session)
|
||||
|
||||
for key, value in data.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
|
|
@ -575,8 +588,6 @@ class Channel(ModelHelper, Base):
|
|||
elif isinstance(self.directory, str) and not pathlib.Path(self.directory).is_absolute():
|
||||
self.directory = get_media_directory() / self.directory
|
||||
|
||||
session: Session = Session.object_session(self)
|
||||
|
||||
for download in downloads:
|
||||
url = None
|
||||
frequency = None
|
||||
|
|
@ -608,11 +619,12 @@ class Channel(ModelHelper, Base):
|
|||
config = dict(
|
||||
calculate_duration=self.calculate_duration,
|
||||
directory=str(self.directory),
|
||||
downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads],
|
||||
generate_posters=self.generate_posters,
|
||||
name=self.name,
|
||||
source_id=self.source_id,
|
||||
tag_name=self.tag_name,
|
||||
url=self.url or None,
|
||||
downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads]
|
||||
)
|
||||
return config
|
||||
|
||||
|
|
@ -627,16 +639,18 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
downloads=self.downloads,
|
||||
directory=self.directory,
|
||||
downloads=self.downloads,
|
||||
id=self.id,
|
||||
name=self.name,
|
||||
tag_name=self.tag_name,
|
||||
url=self.url,
|
||||
)
|
||||
return d
|
||||
|
||||
def dict(self, with_statistics: bool = False, with_downloads: bool = True) -> dict:
|
||||
d = super(Channel, self).dict()
|
||||
d['tag_name'] = self.tag_name
|
||||
d['directory'] = \
|
||||
self.directory.relative_to(get_media_directory()) if self.directory else None
|
||||
if with_statistics:
|
||||
|
|
@ -706,10 +720,9 @@ class Channel(ModelHelper, Base):
|
|||
"""Find a Channel with the provided id, raises an exception when no Channel is found.
|
||||
|
||||
@raise UnknownChannel: if the channel can not be found"""
|
||||
channel = Channel.get_by_id(id_, session=session)
|
||||
if not channel:
|
||||
raise UnknownChannel(f'Cannot find channel with id {id_}')
|
||||
return channel
|
||||
if channel := Channel.get_by_id(id_, session=session):
|
||||
return channel
|
||||
raise UnknownChannel(f'Cannot find channel with id {id_}')
|
||||
|
||||
def get_video_entry_by_id(self, video_source_id: str) -> Optional[Dict]:
|
||||
"""Search my info_json for the entry with the provided id."""
|
||||
|
|
@ -758,3 +771,40 @@ class Channel(ModelHelper, Base):
|
|||
raise RuntimeError('Must provide URL to get Channel')
|
||||
channel = session.query(Channel).filter_by(url=url).one_or_none()
|
||||
return channel
|
||||
|
||||
async def move(self, directory: pathlib.Path, session: Session):
|
||||
"""Move the files of this Channel into a new directory."""
|
||||
if not directory.is_dir():
|
||||
raise FileNotFoundError(f'Destination directory does not exist: {directory}')
|
||||
if self.directory and not next(self.directory.iterdir(), None):
|
||||
raise FileExistsError(f'Refusing to move Channel when it has no files.')
|
||||
|
||||
old_directory = self.directory
|
||||
self.directory = directory
|
||||
session.flush([self, ])
|
||||
|
||||
# Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's
|
||||
# directory.
|
||||
downloads = list(self.downloads)
|
||||
downloads.extend(list(
|
||||
session.query(Download).filter(Download.settings.contains({'destination': str(old_directory)}))
|
||||
))
|
||||
for download in downloads:
|
||||
# Get new copy of `settings` to avoid bug where it wouldn't update.
|
||||
settings = download.settings.copy() if download.settings else dict()
|
||||
if settings.get('destination'):
|
||||
download.settings = {**settings, 'destination': str(directory)}
|
||||
|
||||
session.commit()
|
||||
|
||||
# Save configs before move, this is because move imports configs.
|
||||
from modules.videos.lib import save_channels_config
|
||||
await save_downloads_config(session)
|
||||
save_channels_config(session)
|
||||
save_tags_config(session)
|
||||
|
||||
# Move the contents of the Channel directory into the destination directory.
|
||||
await move(directory, *list(old_directory.iterdir()))
|
||||
|
||||
if old_directory.exists() and not next(old_directory.iterdir(), None):
|
||||
old_directory.rmdir()
|
||||
|
|
|
|||
|
|
@ -37,6 +37,21 @@ class ChannelPutRequest:
|
|||
self.url = self.url.strip() if self.url else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelTagRequest:
|
||||
tag_name: Optional[str] = None
|
||||
directory: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.directory and self.directory.startswith('/'):
|
||||
raise ValidationError('Directory must be relative to media directory')
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelTagInfoRequest:
|
||||
tag_name: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelDownloadRequest:
|
||||
url: str
|
||||
|
|
|
|||
|
|
@ -332,7 +332,7 @@ def test_get_videos_directory(test_directory):
|
|||
def test_get_custom_videos_directory(test_directory, test_config):
|
||||
"""Custom directory can be used for videos directory."""
|
||||
config = get_wrolpi_config()
|
||||
config.videos_directory = 'custom/directory/videos'
|
||||
config.videos_destination = 'custom/directory/videos'
|
||||
|
||||
assert common.get_videos_directory() == (test_directory / 'custom/directory/videos')
|
||||
assert (test_directory / 'custom/directory/videos').is_dir()
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ example_video_json = {
|
|||
'categories': ['Howto & Style'],
|
||||
'channel': 'channel name',
|
||||
'channel_id': 'channel id',
|
||||
'channel_url': 'channel url',
|
||||
'channel_url': 'https://example.com/example_video_json_channel',
|
||||
'duration': 635,
|
||||
'extractor': 'youtube',
|
||||
'extractor_key': 'Youtube',
|
||||
|
|
@ -31,7 +31,7 @@ example_video_json = {
|
|||
'upload_date': '20190707',
|
||||
'uploader': 'uploader name',
|
||||
'uploader_id': 'uploader id',
|
||||
'uploader_url': 'uploader url',
|
||||
'uploader_url': 'https://example.com/example_video_json_uploader',
|
||||
'view_count': 406,
|
||||
'webpage_url': 'webpage url',
|
||||
'webpage_url_basename': 'watch',
|
||||
|
|
|
|||
|
|
@ -5,8 +5,9 @@ import pytest
|
|||
|
||||
from modules.videos import lib
|
||||
from modules.videos.downloader import ChannelDownloader, VideoDownloader
|
||||
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
|
||||
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics, format_videos_destination
|
||||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.downloader import Download, RSSDownloader
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -20,7 +21,7 @@ from wrolpi.vars import PROJECT_DIR
|
|||
('20000101foo.mp4', (None, None, None, '20000101foo')),
|
||||
('something 20000101 foo.mp4', (None, None, None, 'something 20000101 foo')),
|
||||
('something_20000101_foo.mp4', ('something', '20000101', None, 'foo')),
|
||||
('foo .mp4', (None, None, None, 'foo')), # Has trailing whitespace.
|
||||
('foo .mp4', (None, None, None, 'foo')), # Has trailing whitespace.
|
||||
('NA_20000303_vp91w5_Bob's Pancakes.mp4', (None, '20000303', 'vp91w5', 'Bob's Pancakes')),
|
||||
('NA_NA_vp91w5_Bob's Pancakes.mp4', (None, None, 'vp91w5', 'Bob's Pancakes')),
|
||||
('Learning Self-Reliance_20170529_p_MzsCFkUPU_Beekeeping 2017 Part 6 - Merging Hives.mp4',
|
||||
|
|
@ -216,3 +217,33 @@ def test_link_channel_and_downloads_migration(test_async_client, test_session, c
|
|||
assert d2b.channel_id
|
||||
assert d2c.url == 'https://example.com/channel2/video/1' and d2c.frequency is None
|
||||
assert not d2c.channel_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_format_videos_destination(test_async_client, test_directory):
|
||||
"""Videos destination is formatted according to the WROLPiConfig."""
|
||||
wrolpi_config = get_wrolpi_config()
|
||||
|
||||
# Channel directory without a tag.
|
||||
wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_name)s'
|
||||
assert format_videos_destination('Simple Channel', None, None) \
|
||||
== test_directory / 'videos/Simple Channel'
|
||||
|
||||
# One tag can be applied to a Channel.
|
||||
assert format_videos_destination('Simple Channel', 'one', None) \
|
||||
== test_directory / 'videos/one/Simple Channel'
|
||||
|
||||
# Channel Domain is also supported.
|
||||
wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s'
|
||||
assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \
|
||||
== test_directory / 'videos/one/example.com/Simple Channel'
|
||||
|
||||
# Channel name is not required.
|
||||
wrolpi_config.videos_destination = '%(channel_tag)s/%(channel_domain)s'
|
||||
assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \
|
||||
== test_directory / 'one/example.com'
|
||||
|
||||
# Invalid `videos_destination` raises an error.
|
||||
wrolpi_config.videos_destination = '%(channel_tag)s/%(channel)s'
|
||||
with pytest.raises(FileNotFoundError):
|
||||
assert format_videos_destination()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.common import get_relative_to_media_directory, get_wrolpi_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_ffprobe_stream_methods(simple_video):
|
||||
|
|
@ -13,3 +18,24 @@ async def test_ffprobe_stream_methods(simple_video):
|
|||
assert simple_video.get_streams_by_codec_name('h264')
|
||||
assert simple_video.get_streams_by_codec_name('aac')
|
||||
assert simple_video.get_streams_by_codec_name('mov_text')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_move(test_async_client, test_session, test_directory, channel_factory, video_factory):
|
||||
"""A Channel can be moved to another directory, any files in the Channel's directory are moved."""
|
||||
channel = channel_factory(name='Channel Name')
|
||||
video = video_factory(title='Vid', channel_id=channel.id)
|
||||
test_session.commit()
|
||||
assert video.channel_id == channel.id
|
||||
assert str(get_relative_to_media_directory(video.video_path)) == 'videos/Channel Name/Vid.mp4'
|
||||
|
||||
foo = test_directory / 'foo/New Channel Directory'
|
||||
foo.mkdir(parents=True)
|
||||
await channel.move(foo, test_session)
|
||||
assert channel.directory == foo
|
||||
assert channel.directory != (test_directory / 'New Channel Directory')
|
||||
video = test_session.query(Video).one()
|
||||
assert str(get_relative_to_media_directory(video.video_path)) == 'foo/New Channel Directory/Vid.mp4'
|
||||
assert str(video.video_path).startswith(str(channel.directory))
|
||||
# Old directory is deleted.
|
||||
assert not (test_directory / 'Channel Name').exists()
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ def search_videos(
|
|||
joins = list()
|
||||
join_video = False
|
||||
|
||||
params = dict(search_str=search_str, offset=offset)
|
||||
params = dict(search_str=search_str, offset=offset or 0)
|
||||
if channel_id:
|
||||
wheres.append('v.channel_id = %(channel_id)s')
|
||||
joins.append('LEFT JOIN channel c ON c.id = v.channel_id')
|
||||
|
|
|
|||
|
|
@ -464,7 +464,7 @@ def zim_download_url_to_name(url: str) -> Tuple[str, str]:
|
|||
|
||||
|
||||
def get_zim_directory() -> pathlib.Path:
|
||||
zim_directory = get_media_directory() / get_wrolpi_config().zims_directory
|
||||
zim_directory = get_media_directory() / get_wrolpi_config().zims_destination
|
||||
return zim_directory
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -385,6 +385,6 @@ def test_get_custom_zims_directory(test_directory, test_config):
|
|||
"""Custom directory can be used for zims directory."""
|
||||
assert lib.get_zim_directory() == (test_directory / 'zims')
|
||||
|
||||
get_wrolpi_config().zims_directory = 'custom/zims'
|
||||
get_wrolpi_config().zims_destination = 'custom/zims'
|
||||
|
||||
assert lib.get_zim_directory() == (test_directory / 'custom/zims')
|
||||
|
|
|
|||
|
|
@ -27,4 +27,4 @@ srt==3.5.3
|
|||
vininfo[cli]==1.8.0
|
||||
websockets==12.0
|
||||
webvtt-py==0.4.6
|
||||
yt-dlp==2024.8.1
|
||||
yt-dlp==2024.8.6
|
||||
|
|
@ -10,7 +10,7 @@ import tempfile
|
|||
|
||||
from modules.archive import lib as archive_lib
|
||||
from wrolpi.cmd import READABILITY_BIN
|
||||
from wrolpi.common import html_screenshot, format_html_string, logger
|
||||
from wrolpi.common import html_screenshot, format_html_string, logger, format_html_file
|
||||
|
||||
logger = logger.getChild('singlefile_to_archive')
|
||||
|
||||
|
|
@ -31,6 +31,8 @@ def singlefile_to_archive_files(singlefile: pathlib.Path):
|
|||
"""Extract Readability files and Screenshot file next to the provided Singlefile."""
|
||||
contents = singlefile.read_bytes()
|
||||
|
||||
format_html_file(singlefile)
|
||||
|
||||
readability_html_path = singlefile.with_suffix('.readability.html')
|
||||
readability_json_path = singlefile.with_suffix('.readability.json')
|
||||
readability_text_path = singlefile.with_suffix('.readability.txt')
|
||||
|
|
|
|||
|
|
@ -276,7 +276,10 @@ class ModelHelper:
|
|||
|
||||
def flush(self):
|
||||
"""A convenience function which flushes this record using its DB Session."""
|
||||
Session.object_session(self).flush([self])
|
||||
if session := Session.object_session(self):
|
||||
session.flush([self])
|
||||
else:
|
||||
raise RuntimeError(f'{self} is not in a session!')
|
||||
|
||||
|
||||
class tsvector(types.TypeDecorator):
|
||||
|
|
@ -370,7 +373,8 @@ class ConfigFile:
|
|||
if config_file.is_file():
|
||||
# Use the config file to get the values the user set.
|
||||
with config_file.open('rt') as fh:
|
||||
self._config.update(yaml.load(fh, Loader=yaml.Loader))
|
||||
config_data = yaml.load(fh, Loader=yaml.Loader)
|
||||
self._config.update(config_data)
|
||||
return self
|
||||
|
||||
def _get_backup_filename(self):
|
||||
|
|
@ -467,7 +471,7 @@ class ConfigFile:
|
|||
class WROLPiConfig(ConfigFile):
|
||||
file_name = 'wrolpi.yaml'
|
||||
default_config = dict(
|
||||
archive_directory='archive',
|
||||
archive_destination='archive/%(domain)s',
|
||||
download_on_startup=True,
|
||||
download_timeout=0,
|
||||
hotspot_device='wlan0',
|
||||
|
|
@ -476,12 +480,12 @@ class WROLPiConfig(ConfigFile):
|
|||
hotspot_ssid='WROLPi',
|
||||
ignore_outdated_zims=False,
|
||||
ignored_directories=['config', 'tags'],
|
||||
map_directory='map',
|
||||
map_destination='map',
|
||||
nav_color='violet',
|
||||
throttle_on_startup=False,
|
||||
videos_directory='videos',
|
||||
videos_destination='videos/%(channel_tag)s/%(channel_name)s',
|
||||
wrol_mode=False,
|
||||
zims_directory='zims',
|
||||
zims_destination='zims',
|
||||
)
|
||||
|
||||
def get_file(self) -> Path:
|
||||
|
|
@ -586,28 +590,36 @@ class WROLPiConfig(ConfigFile):
|
|||
self.update({'ignored_directories': value})
|
||||
|
||||
@property
|
||||
def videos_directory(self) -> str:
|
||||
return self._config['videos_directory']
|
||||
def videos_destination(self) -> str:
|
||||
return self._config['videos_destination']
|
||||
|
||||
@videos_directory.setter
|
||||
def videos_directory(self, value: str):
|
||||
self.update({'videos_directory': value})
|
||||
@videos_destination.setter
|
||||
def videos_destination(self, value: str):
|
||||
self.update({'videos_destination': value})
|
||||
|
||||
@property
|
||||
def archive_directory(self) -> str:
|
||||
return self._config['archive_directory']
|
||||
def archive_destination(self) -> str:
|
||||
return self._config['archive_destination']
|
||||
|
||||
@archive_directory.setter
|
||||
def archive_directory(self, value: str):
|
||||
self.update({'archive_directory': value})
|
||||
@archive_destination.setter
|
||||
def archive_destination(self, value: str):
|
||||
self.update({'archive_destination': value})
|
||||
|
||||
@property
|
||||
def map_directory(self) -> str:
|
||||
return self._config['map_directory']
|
||||
def map_destination(self) -> str:
|
||||
return self._config['map_destination']
|
||||
|
||||
@map_directory.setter
|
||||
def map_directory(self, value: str):
|
||||
self.update({'map_directory': value})
|
||||
@map_destination.setter
|
||||
def map_destination(self, value: str):
|
||||
self.update({'map_destination': value})
|
||||
|
||||
@property
|
||||
def zims_destination(self) -> str:
|
||||
return self._config['zims_destination']
|
||||
|
||||
@zims_destination.setter
|
||||
def zims_destination(self, value: str):
|
||||
self.update({'zims_destination': value})
|
||||
|
||||
@property
|
||||
def nav_color(self) -> str:
|
||||
|
|
@ -617,14 +629,6 @@ class WROLPiConfig(ConfigFile):
|
|||
def nav_color(self, value: str):
|
||||
self.update({'nav_color': value})
|
||||
|
||||
@property
|
||||
def zims_directory(self) -> str:
|
||||
return self._config['zims_directory']
|
||||
|
||||
@zims_directory.setter
|
||||
def zims_directory(self, value: str):
|
||||
self.update({'zims_directory': value})
|
||||
|
||||
|
||||
WROLPI_CONFIG: WROLPiConfig = WROLPiConfig()
|
||||
TEST_WROLPI_CONFIG: WROLPiConfig = None
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import wrolpi.root_api # noqa
|
|||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode
|
||||
from wrolpi.common import logger
|
||||
from wrolpi.common import set_test_media_directory, Base, set_test_config
|
||||
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
|
||||
from wrolpi.dates import set_test_now
|
||||
|
|
@ -47,6 +48,8 @@ from wrolpi.files.models import Directory, FileGroup
|
|||
from wrolpi.tags import Tag, upsert_tag
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
def get_test_db_engine():
|
||||
suffix = str(uuid1()).replace('-', '')
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from wrolpi import flags
|
|||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.cmd import pid_is_running
|
||||
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \
|
||||
limit_concurrent, wrol_mode_enabled
|
||||
limit_concurrent, wrol_mode_enabled, background_task
|
||||
from wrolpi.dates import TZDateTime, now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload
|
||||
|
|
@ -226,7 +226,7 @@ class Download(ModelHelper, Base): # noqa
|
|||
from modules.videos.lib import save_channels_config
|
||||
save_channels_config()
|
||||
# Save download config again because this download is now removed from the download lists.
|
||||
api_app.add_task(save_downloads_config())
|
||||
background_task(save_downloads_config())
|
||||
|
||||
@staticmethod
|
||||
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
|
||||
|
|
@ -538,9 +538,9 @@ class DownloadManager:
|
|||
session.flush(downloads)
|
||||
try:
|
||||
# Start downloading ASAP.
|
||||
api_app.add_task(self.dispatch_downloads())
|
||||
background_task(self.dispatch_downloads())
|
||||
# Save the config now that new Downloads exist.
|
||||
api_app.add_task(save_downloads_config())
|
||||
background_task(save_downloads_config())
|
||||
except RuntimeError:
|
||||
# Event loop isn't running. Probably testing?
|
||||
if not PYTEST:
|
||||
|
|
@ -574,9 +574,6 @@ class DownloadManager:
|
|||
from modules.videos.models import Channel
|
||||
if channel := Channel.get_by_url(url=download.url, session=session):
|
||||
download.channel_id = channel.id
|
||||
if destination := (download.settings or dict()).get('destination'):
|
||||
if channel := Channel.get_by_path(destination, session):
|
||||
download.channel_id = channel.id
|
||||
|
||||
session.commit()
|
||||
|
||||
|
|
@ -668,7 +665,7 @@ class DownloadManager:
|
|||
with get_db_curs(commit=True) as curs:
|
||||
curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
|
||||
|
||||
api_app.add_task(save_downloads_config())
|
||||
background_task(save_downloads_config())
|
||||
|
||||
DOWNLOAD_SORT = (
|
||||
DownloadStatus.pending,
|
||||
|
|
@ -742,7 +739,7 @@ class DownloadManager:
|
|||
session.commit()
|
||||
|
||||
# Save the config now that some Downloads renewed.
|
||||
api_app.add_task(save_downloads_config())
|
||||
background_task(save_downloads_config())
|
||||
|
||||
@staticmethod
|
||||
def get_downloads(session: Session) -> List[Download]:
|
||||
|
|
@ -1025,7 +1022,7 @@ class DownloadManager:
|
|||
stmt = self._delete_downloads_q(once=True)
|
||||
deleted_ids = [i for i, in session.execute(stmt).fetchall()]
|
||||
session.commit()
|
||||
api_app.add_task(save_downloads_config())
|
||||
background_task(save_downloads_config())
|
||||
return deleted_ids
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -1130,7 +1127,7 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
# Allow the download to resume.
|
||||
download_manager.unkill_download(download_id)
|
||||
# Save the config now that the Download has finished.
|
||||
api_app.add_task(save_downloads_config())
|
||||
background_task(save_downloads_config())
|
||||
except asyncio.CancelledError as e:
|
||||
worker_logger.warning('Canceled!', exc_info=e)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ def get_settings(_: Request):
|
|||
ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories]
|
||||
|
||||
settings = {
|
||||
'archive_directory': config.archive_directory,
|
||||
'archive_directory': config.archive_destination,
|
||||
'download_manager_disabled': download_manager.is_disabled,
|
||||
'download_manager_stopped': download_manager.is_stopped,
|
||||
'download_on_startup': config.download_on_startup,
|
||||
|
|
@ -150,15 +150,15 @@ def get_settings(_: Request):
|
|||
'ignore_outdated_zims': config.ignore_outdated_zims,
|
||||
'ignored_directories': ignored_directories,
|
||||
'log_level': api_app.shared_ctx.log_level.value,
|
||||
'map_directory': config.map_directory,
|
||||
'map_directory': config.map_destination,
|
||||
'nav_color': config.nav_color,
|
||||
'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative.
|
||||
'throttle_on_startup': config.throttle_on_startup,
|
||||
'throttle_status': admin.throttle_status().name,
|
||||
'version': __version__,
|
||||
'videos_directory': config.videos_directory,
|
||||
'videos_directory': config.videos_destination,
|
||||
'wrol_mode': config.wrol_mode,
|
||||
'zims_directory': config.zims_directory,
|
||||
'zims_directory': config.zims_destination,
|
||||
}
|
||||
return json_response(settings)
|
||||
|
||||
|
|
@ -190,25 +190,25 @@ async def update_settings(_: Request, body: schema.SettingsRequest):
|
|||
if not config:
|
||||
raise InvalidConfig()
|
||||
|
||||
if body.archive_directory and pathlib.Path(body.archive_directory).is_absolute():
|
||||
if body.archive_destination and pathlib.Path(body.archive_destination).is_absolute():
|
||||
raise InvalidConfig('Archive directory must be relative to media directory')
|
||||
elif not body.archive_directory:
|
||||
config['archive_directory'] = wrolpi_config.default_config['archive_directory']
|
||||
elif not body.archive_destination:
|
||||
config['archive_destination'] = wrolpi_config.default_config['archive_destination']
|
||||
|
||||
if body.videos_directory and pathlib.Path(body.videos_directory).is_absolute():
|
||||
if body.videos_destination and pathlib.Path(body.videos_destination).is_absolute():
|
||||
raise InvalidConfig('Videos directory must be relative to media directory')
|
||||
elif not body.videos_directory:
|
||||
config['videos_directory'] = wrolpi_config.default_config['videos_directory']
|
||||
elif not body.videos_destination:
|
||||
config['videos_destination'] = wrolpi_config.default_config['videos_destination']
|
||||
|
||||
if body.map_directory and pathlib.Path(body.map_directory).is_absolute():
|
||||
if body.map_destination and pathlib.Path(body.map_destination).is_absolute():
|
||||
raise InvalidConfig('Map directory must be relative to media directory')
|
||||
elif not body.map_directory:
|
||||
config['map_directory'] = wrolpi_config.default_config['map_directory']
|
||||
elif not body.map_destination:
|
||||
config['map_destination'] = wrolpi_config.default_config['map_destination']
|
||||
|
||||
if body.zims_directory and pathlib.Path(body.zims_directory).is_absolute():
|
||||
if body.zims_destination and pathlib.Path(body.zims_destination).is_absolute():
|
||||
raise InvalidConfig('Zims directory must be relative to media directory')
|
||||
elif not body.zims_directory:
|
||||
config['zims_directory'] = wrolpi_config.default_config['zims_directory']
|
||||
elif not body.zims_destination:
|
||||
config['zims_destination'] = wrolpi_config.default_config['zims_destination']
|
||||
|
||||
log_level = config.pop('log_level', None)
|
||||
if isinstance(log_level, int):
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class SemanticUIColors(enum.StrEnum):
|
|||
|
||||
@dataclass
|
||||
class SettingsRequest:
|
||||
archive_directory: Optional[str] = None
|
||||
archive_destination: Optional[str] = None
|
||||
download_on_startup: Optional[bool] = None
|
||||
download_timeout: Optional[int] = None
|
||||
hotspot_device: Optional[str] = None
|
||||
|
|
@ -42,14 +42,14 @@ class SettingsRequest:
|
|||
hotspot_status: Optional[bool] = None
|
||||
ignore_outdated_zims: Optional[bool] = None
|
||||
log_level: Optional[int] = None
|
||||
map_directory: Optional[str] = None
|
||||
map_destination: Optional[str] = None
|
||||
nav_color: Optional[str] = None
|
||||
media_directory: Optional[str] = None
|
||||
throttle_on: Optional[bool] = None
|
||||
throttle_on_startup: Optional[bool] = None
|
||||
videos_directory: Optional[str] = None
|
||||
videos_destination: Optional[str] = None
|
||||
wrol_mode: Optional[bool] = None
|
||||
zims_directory: Optional[str] = None
|
||||
zims_destination: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.nav_color = self.nav_color.lower() if self.nav_color else None
|
||||
|
|
|
|||
|
|
@ -426,7 +426,6 @@ def delete_tag(tag_id: int, session: Session = None):
|
|||
sync_tags_directory()
|
||||
|
||||
|
||||
@register_refresh_cleanup
|
||||
@optional_session
|
||||
def import_tags_config(session: Session = None):
|
||||
"""Reads the Tags and TagFiles from the config file, upserts them in the DB."""
|
||||
|
|
|
|||
|
|
@ -530,14 +530,15 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
|
|||
request, response = await test_async_client.post('/api/search_suggestions', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
if expected_channels:
|
||||
assert response.json['channels'] == expected_channels
|
||||
for channel, expected_channel in zip_longest(response.json['channels'], expected_channels):
|
||||
assert_dict_contains(channel, expected_channel)
|
||||
assert response.json['domains'] == expected_domains
|
||||
|
||||
await assert_results(
|
||||
dict(search_str='foo'),
|
||||
[
|
||||
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []},
|
||||
{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []},
|
||||
{'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []},
|
||||
{'directory': 'videos/Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []},
|
||||
],
|
||||
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}],
|
||||
)
|
||||
|
|
@ -546,14 +547,14 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
|
|||
await assert_results(
|
||||
dict(search_str='foo l'),
|
||||
[
|
||||
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}],
|
||||
{'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}],
|
||||
[],
|
||||
)
|
||||
|
||||
await assert_results(
|
||||
dict(search_str='bar'),
|
||||
[
|
||||
{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []}
|
||||
{'directory': 'videos/Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []}
|
||||
],
|
||||
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}],
|
||||
)
|
||||
|
|
@ -705,29 +706,28 @@ def test_recursive_errors():
|
|||
@pytest.mark.asyncio
|
||||
async def test_settings_special_directories(test_async_client, test_config):
|
||||
"""Maintainer can change special directories."""
|
||||
assert get_wrolpi_config().archive_directory == 'archive'
|
||||
|
||||
data = {'archive_directory': 'archives'}
|
||||
data = {'archive_destination': 'archives'}
|
||||
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
assert get_wrolpi_config().archive_directory == 'archives'
|
||||
assert get_wrolpi_config().archive_destination == 'archives'
|
||||
|
||||
data = {'archive_directory': '/absolute/not/allowed'}
|
||||
data = {'archive_destination': '/absolute/not/allowed'}
|
||||
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
data = {'videos_directory': '/absolute/not/allowed'}
|
||||
data = {'videos_destination': '/absolute/not/allowed'}
|
||||
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
data = {'map_directory': '/absolute/not/allowed'}
|
||||
data = {'map_destination': '/absolute/not/allowed'}
|
||||
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
data = {'zims_directory': '/absolute/not/allowed'}
|
||||
data = {'zims_destination': '/absolute/not/allowed'}
|
||||
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
# Empty directory restores default.
|
||||
data = {'archive_directory': ''}
|
||||
data = {'archive_destination': ''}
|
||||
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert get_wrolpi_config().archive_directory == 'archive'
|
||||
assert get_wrolpi_config().archive_destination == 'archive/%(domain)s'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue