: null}
Videos: {channel.video_count}
@@ -669,6 +756,7 @@ export function ChannelsPage() {
const headers = [
{key: 'name', text: 'Name', sortBy: 'name', width: 8},
+ {key: 'tag', text: 'Tag', sortBy: 'tag', width: 2},
{key: 'video_count', text: 'Videos', sortBy: 'video_count', width: 2},
{key: 'download_frequency', text: 'Download Frequency', sortBy: 'minimum_frequency', width: 2},
{key: 'size', text: 'Size', sortBy: 'size', width: 2},
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js
index beb40eae..c8368885 100644
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@@ -734,6 +734,10 @@ export const useDirectories = (defaultDirectory) => {
fetchDirectories();
}, [directory, defaultDirectory]);
+ useEffect(() => {
+ setDirectory(defaultDirectory);
+ }, [defaultDirectory])
+
return {directory, directories, setDirectory, exists, isDir, isFile};
}
diff --git a/main.py b/main.py
index 1e1afc92..6778adf2 100755
--- a/main.py
+++ b/main.py
@@ -322,9 +322,9 @@ async def perpetual_have_internet_worker():
# Check hourly once we have internet.
await asyncio.sleep(float(Seconds.hour))
else:
- # Check every minute until the internet is back.
+ # Check more often until the internet is back.
flags.have_internet.clear()
- await asyncio.sleep(float(Seconds.minute))
+ await asyncio.sleep(10)
except Exception as e:
logger.error('Failed to check if internet is up', exc_info=e)
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 3131072d..45992410 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -48,7 +48,10 @@ class ArchiveFiles:
def get_archive_directory() -> pathlib.Path:
- archive_directory = get_media_directory() / get_wrolpi_config().archive_directory
+ archive_destination = get_wrolpi_config().archive_destination
+ variables = dict(domain='', year='', month='', day='')
+ archive_destination = archive_destination % variables
+ archive_directory = get_media_directory() / archive_destination
return archive_directory
diff --git a/modules/archive/models.py b/modules/archive/models.py
index 8c41c0c4..05ea88a0 100644
--- a/modules/archive/models.py
+++ b/modules/archive/models.py
@@ -9,7 +9,8 @@ from sqlalchemy.orm import relationship, Session, validates
from sqlalchemy.orm.collections import InstrumentedList
from wrolpi import dates
-from wrolpi.common import ModelHelper, Base, logger, get_title_from_html
+from wrolpi.common import ModelHelper, Base, logger, get_title_from_html, get_wrolpi_config, get_media_directory
+from wrolpi.dates import now
from wrolpi.db import optional_session
from wrolpi.errors import UnknownArchive
from wrolpi.files.models import FileGroup
@@ -342,3 +343,21 @@ class Domain(Base, ModelHelper):
if len(value.split('.')) < 2:
raise ValueError(f'Domain must contain at least one "." domain={repr(value)}')
return value
+
+ @property
+ def download_directory(self) -> pathlib.Path:
+ archive_destination = get_wrolpi_config().archive_destination
+
+ now_ = now()
+ variables = dict(
+ domain=self.domain,
+ year=now_.year,
+ month=now_.month,
+ day=now_.day,
+ )
+ archive_destination = archive_destination % variables
+ archive_destination = pathlib.Path(archive_destination.lstrip('/'))
+ if not archive_destination.is_absolute():
+ archive_destination = get_media_directory() / archive_destination
+
+ return archive_destination
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index d9634c30..ceb254b6 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -820,12 +820,13 @@ def test_get_url_from_singlefile():
assert lib.get_url_from_singlefile(SINGLEFILE_EXAMPLE_1) == 'https://www.example.com'
-def test_get_custom_archive_directory(test_directory, test_config):
+@pytest.mark.asyncio
+async def test_get_custom_archive_directory(test_async_client, test_directory, test_config):
"""Custom directory can be used for archive directory."""
# Default location.
assert lib.get_archive_directory() == (test_directory / 'archive')
- get_wrolpi_config().archive_directory = 'custom/archives'
+ get_wrolpi_config().archive_destination = 'custom/archives'
assert lib.get_archive_directory() == (test_directory / 'custom/archives')
assert lib.get_domain_directory('https://example.com') == (test_directory / 'custom/archives/example.com')
diff --git a/modules/archive/test/test_models.py b/modules/archive/test/test_models.py
new file mode 100644
index 00000000..c580d6d1
--- /dev/null
+++ b/modules/archive/test/test_models.py
@@ -0,0 +1,27 @@
+from datetime import datetime
+
+import pytest
+
+from modules.archive import Domain
+from wrolpi.common import get_wrolpi_config
+
+
+@pytest.mark.asyncio
+async def test_archive_download_destination(test_async_client, test_session, test_directory, archive_factory, fake_now):
+ fake_now(datetime(2000, 1, 2))
+
+ wrolpi_config = get_wrolpi_config()
+
+ archive_factory(domain='wrolpi.org')
+ domain = test_session.query(Domain).one()
+
+ # Test the default download directory.
+ assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org')
+
+ # Year of download is supported.
+ wrolpi_config.archive_destination = 'archives/%(domain)s/%(year)s'
+ assert str(domain.download_directory) == str(test_directory / 'archives/wrolpi.org/2000')
+
+ # More download date is supported.
+ wrolpi_config.archive_destination = 'archive/%(domain)s/%(year)s/%(month)s/%(day)s'
+ assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org/2000/1/2')
diff --git a/modules/map/lib.py b/modules/map/lib.py
index 0107b430..f72ba914 100644
--- a/modules/map/lib.py
+++ b/modules/map/lib.py
@@ -19,7 +19,7 @@ logger = logger.getChild(__name__)
def get_map_directory() -> Path:
- map_directory = get_media_directory() / get_wrolpi_config().map_directory
+ map_directory = get_media_directory() / get_wrolpi_config().map_destination
if not map_directory.is_dir():
map_directory.mkdir(parents=True)
return map_directory
diff --git a/modules/map/test/test_lib.py b/modules/map/test/test_lib.py
index 8e31d4cf..a1ed847b 100644
--- a/modules/map/test/test_lib.py
+++ b/modules/map/test/test_lib.py
@@ -107,7 +107,7 @@ def test_get_custom_map_directory(test_directory, test_config):
# Default location.
assert lib.get_map_directory() == (test_directory / 'map')
- get_wrolpi_config().map_directory = 'custom/deep/map/directory'
+ get_wrolpi_config().map_destination = 'custom/deep/map/directory'
assert lib.get_map_directory() == (test_directory / 'custom/deep/map/directory')
assert (test_directory / 'custom/deep/map/directory').is_dir()
diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py
index ea259a0e..4bc5ade3 100644
--- a/modules/videos/channel/api.py
+++ b/modules/videos/channel/api.py
@@ -17,7 +17,7 @@ from wrolpi.vars import PYTEST
from . import lib
from .. import schema
from ..errors import UnknownChannel
-from ..lib import save_channels_config
+from ..lib import save_channels_config, format_videos_destination
from ..models import Channel
channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels')
@@ -136,3 +136,41 @@ async def channel_download(_: Request, channel_id: int, body: schema.ChannelDown
# Warn the user that downloads are disabled.
Events.send_downloads_disabled('Channel download created. But downloads are disabled.')
return response.empty()
+
+
+@channel_bp.post('//tag_info')
+@openapi.definition(
+ description='Get Channel Tag Info',
+ body=schema.ChannelTagInfoRequest,
+)
+@validate(schema.ChannelTagInfoRequest)
+@run_after(save_channels_config)
+@wrol_mode_check
+async def channel_tag_info(_: Request, channel_id: int, body: schema.ChannelTagInfoRequest):
+ from wrolpi.tags import Tag
+ channel_name = Channel.find_by_id(channel_id).name
+ tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None
+ videos_destination = format_videos_destination(channel_name, tag_name)
+ ret = dict(videos_destination=videos_destination)
+ return json_response(ret)
+
+
+@channel_bp.post('//tag')
+@openapi.definition(
+ description='Tag Channel',
+ body=schema.ChannelTagRequest,
+)
+@openapi.response(HTTPStatus.NO_CONTENT)
+@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
+@validate(schema.ChannelTagRequest)
+@run_after(save_channels_config)
+@wrol_mode_check
+async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagRequest):
+ directory = None
+ if body.directory:
+ directory = pathlib.Path(body.directory)
+ directory = get_media_directory() / directory
+
+ await lib.tag_channel(body.tag_name, directory, channel_id)
+
+ return response.raw('', HTTPStatus.NO_CONTENT)
diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index bba55415..055658b4 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -1,9 +1,9 @@
+import pathlib
from pathlib import Path
from typing import List, Dict, Union
from sqlalchemy import or_, func, desc, asc
from sqlalchemy.orm import Session
-from sqlalchemy.orm.exc import NoResultFound
from wrolpi.common import run_after, logger, \
get_media_directory
@@ -12,7 +12,7 @@ from wrolpi.downloader import save_downloads_config, download_manager
from wrolpi.errors import UnknownDirectory, APIError, ValidationError
from .. import schema
from ..common import check_for_channel_conflicts
-from ..errors import UnknownChannel
+from ..errors import UnknownChannel, ChannelDirectoryConflict
from ..lib import save_channels_config
from ..models import Channel, Video
@@ -28,7 +28,7 @@ async def get_minimal_channels() -> List[dict]:
# one that will consume the most resources.
stmt = '''
SELECT
- c.id, name, directory, c.url,
+ c.id, c.name AS "name", directory, c.url, t.name AS "tag_name",
COUNT(v.id) as video_count,
SUM(fg.size)::BIGINT AS size,
(
@@ -40,7 +40,8 @@ async def get_minimal_channels() -> List[dict]:
channel AS c
LEFT JOIN video v on c.id = v.channel_id
LEFT JOIN file_group fg on fg.id = v.file_group_id
- GROUP BY 1, 2, 3, 4
+ LEFT JOIN tag t ON t.id = c.tag_id
+ GROUP BY 1, 2, 3, 4, 5
'''
curs.execute(stmt)
channels = sorted([dict(i) for i in curs.fetchall()], key=lambda i: i['name'].lower())
@@ -115,10 +116,7 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No
@optional_session
def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
"""Update a Channel's DB record"""
- try:
- channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
- except NoResultFound:
- raise UnknownChannel()
+ channel = Channel.find_by_id(channel_id, session)
# Only update directory if it was empty
if data.directory and not channel.directory:
@@ -237,8 +235,9 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
Channel.find_by_id(channel_id, session=session)
download = download_manager.get_download(session, id_=download_id)
download.url = url
- download.frequency = download.frequency or frequency
+ download.frequency = frequency
download.settings = settings
+ session.commit()
download_manager.remove_from_skip_list(url)
@@ -246,3 +245,38 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
await save_downloads_config()
return download
+
+
+@optional_session
+async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id: int, session: Session = None):
+ """Add a Tag to a Channel, or remove a Tag from a Channel if no `tag_name` is provided.
+
+ Move the Channel to the new directory, if provided."""
+ from wrolpi.tags import Tag
+
+ channel = Channel.find_by_id(channel_id, session=session)
+
+ if tag_name:
+ tag = Tag.find_by_name(tag_name, session=session)
+ channel.tag_id = tag.id
+ channel.tag = tag
+ else:
+ channel.tag = channel.tag_id = None
+
+ session.flush([channel, ])
+
+ # Do not move Channel files into a directory with files.
+ if directory and directory.is_dir() and next(directory.iterdir(), None):
+ raise ChannelDirectoryConflict('Channel directory already exists and is not empty')
+
+ # Only move Channel when requested.
+ if directory:
+ # Move to newly defined directory only if necessary.
+ logger.info(f'Tagging {channel} with {tag_name} moving to {directory} from {channel.directory}')
+ directory.mkdir(parents=True, exist_ok=True)
+ if channel.directory != directory:
+ await channel.move(directory, session)
+ else:
+ logger.info(f'Tagging {channel} with {tag_name}')
+
+ session.commit()
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index ea8fdd60..c615bb4f 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -8,8 +8,11 @@ from http import HTTPStatus
import mock
import pytest
+from modules.videos.common import get_videos_directory
from modules.videos.downloader import ChannelDownloader
-from modules.videos.models import Channel
+from modules.videos.lib import save_channels_config, get_channels_config
+from modules.videos.models import Channel, Video
+from wrolpi.common import get_relative_to_media_directory, walk
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import download_manager, Download, DownloadResult
@@ -412,7 +415,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
@pytest.mark.asyncio
-async def test_create_channel_download(test_async_client, test_session, simple_channel, tag_factory):
+async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory):
tag = tag_factory()
# Create Download.
@@ -452,3 +455,93 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 1
assert test_session.query(Channel).count() == 1
+
+
+@pytest.mark.asyncio
+async def test_tag_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory, video_factory,
+ test_channels_config, test_download_manager, test_downloader):
+ """A single Tag can be applied to a Channel."""
+ # Create channel directory in the usual videos directory.
+ videos_directory = get_videos_directory()
+ channel = channel_factory(name='Channel Name', download_frequency=120)
+ channel_directory = channel.directory
+ tag = tag_factory('Tag Name')
+ v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
+ # Create recurring download which uses the Channel's directory.
+ download = download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
+ settings={
+ 'destination': str(test_directory / 'videos/Channel Name')})
+ test_session.commit()
+ save_channels_config()
+ assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Channel Name')
+ # Channel download downloads into the Channel's directory.
+ assert channel.downloads[0].settings['destination'] == str(test_directory / 'videos/Channel Name')
+ # Make extra file in the Channel's directory, it should be moved.
+ (channel_directory / 'extra file.txt').write_text('extra file contents')
+ # Channel directory is in the Videos directory.
+ assert channel.directory == channel_directory
+ # Videos are in the Channel's Directory.
+ assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4'
+ assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4'
+ video_files = [i for i in walk(videos_directory) if i.is_file()]
+ assert len(video_files) == 3
+
+ body = dict(tag_name=tag.name, directory='videos/Tag Name/Channel Name')
+ request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
+ assert response.status_code == HTTPStatus.NO_CONTENT
+
+ channel = test_session.query(Channel).one()
+ assert channel.tag_name == tag.name
+ # Channel was moved to Tag's directory, old directory was removed
+ new_channel_directory = test_directory / 'videos/Tag Name/Channel Name'
+ assert channel.directory == new_channel_directory, 'Channel directory should have been changed.'
+ assert new_channel_directory.is_dir(), 'Channel should have been moved.'
+ assert next(new_channel_directory.iterdir(), None), 'Channel videos should have been moved.'
+ assert not channel_directory.exists(), 'Old Channel directory should have been deleted.'
+ # Channel download goes into the Channel's directory.
+ d1, d2 = test_session.query(Download).all()
+ assert d1.settings['destination'] == str(test_directory / 'videos/Tag Name/Channel Name'), \
+ f'{d1} was not moved'
+ assert d2.settings['destination'] == str(test_directory / 'videos/Tag Name/Channel Name'), \
+ f'{d2} was not moved'
+
+ # Videos were moved
+ v1, v2 = test_session.query(Video).order_by(Video.id).all()
+ assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Tag Name/Channel Name/video1.mp4'
+ assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Tag Name/Channel Name/video2.mp4'
+ # Extra file was also moved.
+ assert (new_channel_directory / 'extra file.txt').read_text() == 'extra file contents'
+ # No new files were created.
+ assert len([i for i in walk(videos_directory) if i.is_file()]) == 3
+
+ assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Tag Name/Channel Name')
+
+ # Remove tag, Channel/Videos should be moved back.
+ body = dict(tag_name=None, directory='videos/Channel Name')
+ request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
+ channel = test_session.query(Channel).one()
+ assert channel.tag_name is None
+ assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
+ v1, v2 = test_session.query(Video).order_by(Video.id).all()
+ assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4'
+ assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4'
+ assert not (test_directory / 'videos/Tag Name/Channel Name').exists()
+ # Downloads are moved back.
+ d1, d2 = test_session.query(Download).all()
+ assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
+ assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
+
+ # Channel can be Tagged, but not move directories.
+ body = dict(tag_name=tag.name)
+ request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
+ channel = test_session.query(Channel).one()
+ assert channel.tag_name == 'Tag Name'
+ assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
+ assert str(get_relative_to_media_directory(v1.video_path)) == 'videos/Channel Name/video1.mp4'
+ assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4'
+ # Downloads were not changed.
+ d1, d2 = test_session.query(Download).all()
+ assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
+ assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
diff --git a/modules/videos/channel/test/test_models.py b/modules/videos/channel/test/test_models.py
index 3668db42..80deab14 100644
--- a/modules/videos/channel/test/test_models.py
+++ b/modules/videos/channel/test/test_models.py
@@ -3,10 +3,9 @@ from http import HTTPStatus
import pytest
-from modules.videos.downloader import VideoDownloader
from modules.videos.models import Channel
from modules.videos.models import Video
-from wrolpi.downloader import Download, DownloadFrequency, download_manager, RSSDownloader
+from wrolpi.downloader import Download, DownloadFrequency
def test_delete_channel_no_url(test_session, test_client, channel_factory):
@@ -102,17 +101,3 @@ async def test_channel_download_relationships(test_session, download_channel):
# Delete the Channel, and any relationships.
download_channel.delete_with_videos()
assert test_session.query(Download).count() == 0
-
-
-@pytest.mark.asyncio()
-async def test_create_channel_download(test_session, simple_channel):
- """A Download will be related to a Channel if a Download is created in the Channel's directory"""
- test_session.flush()
-
- settings = dict(destination=str(simple_channel.directory))
- download_manager.recurring_download('https://example.com/2', 1, RSSDownloader.name, test_session,
- sub_downloader_name=VideoDownloader.name, settings=settings)
- download = test_session.query(Download).one()
- assert download.settings['destination'] == str(simple_channel.directory)
- download = test_session.query(Download).one()
- assert download.channel == simple_channel
diff --git a/modules/videos/common.py b/modules/videos/common.py
index c63e0745..8a4307ef 100644
--- a/modules/videos/common.py
+++ b/modules/videos/common.py
@@ -35,8 +35,9 @@ class ConfigError(Exception):
def get_videos_directory() -> pathlib.Path:
"""Get the "videos" directory in the media directory. Make it if it does not exist."""
- config = get_wrolpi_config()
- directory = get_media_directory() / config.videos_directory
+ from modules.videos.lib import format_videos_destination
+ videos_destination = format_videos_destination()
+ directory = get_media_directory() / videos_destination
if not directory.is_dir():
directory.mkdir(parents=True)
return directory
diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py
index 0b417f96..0e2f7240 100644
--- a/modules/videos/conftest.py
+++ b/modules/videos/conftest.py
@@ -11,12 +11,11 @@ import pytest
from PIL import Image
from modules.videos.downloader import VideoDownloader, ChannelDownloader
-from modules.videos.lib import set_test_channels_config, set_test_downloader_config
+from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination
from modules.videos.models import Channel, Video
from wrolpi.api_utils import api_app
from wrolpi.downloader import DownloadFrequency, DownloadManager
from wrolpi.files.models import FileGroup
-from wrolpi.tags import Tag
from wrolpi.vars import PROJECT_DIR
@@ -36,25 +35,25 @@ def simple_channel(test_session, test_directory) -> Channel:
@pytest.fixture
def channel_factory(test_session, test_directory):
"""Create a random Channel with a directory, and Download."""
+ from wrolpi.tags import Tag
def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None,
- directory: pathlib.Path = None):
+ directory: pathlib.Path = None, tag_name: str = None):
name = name or str(uuid4())
- directory = directory or test_directory / name
- if not directory.is_dir():
- directory.mkdir()
+ tag = Tag.get_by_name(tag_name) if tag_name else None
channel = Channel(
- directory=directory, # noqa
name=name,
url=url or f'https://example.com/{name}',
source_id=source_id,
+ tag=tag,
)
+ channel.directory = directory or format_videos_destination(name, tag, channel.url)
+ channel.directory.mkdir(exist_ok=True, parents=True)
test_session.add(channel)
test_session.flush([channel])
assert channel.id and channel.url
if download_frequency:
- download = channel.get_or_create_download(channel.url, 60)
- download.frequency = download_frequency
+ download = channel.get_or_create_download(channel.url, download_frequency)
assert download.url == channel.url
assert channel.downloads
test_session.commit()
@@ -106,8 +105,9 @@ def video_factory(test_session, test_directory):
path = pathlib.Path(with_video_file) if not isinstance(with_video_file, pathlib.Path) else with_video_file
elif channel_id:
# Put the video in its Channel's directory.
- channel = test_session.query(Channel).filter_by(id=channel_id).one()
+ channel = Channel.find_by_id(channel_id)
path = (channel.directory or test_directory) / f'{title}.mp4'
+ path.parent.mkdir(exist_ok=True, parents=True)
else:
# Put any video not in a Channel and without a specified file in the NO CHANNEL directory.
(test_directory / 'videos/NO CHANNEL').mkdir(exist_ok=True, parents=True)
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 95346e89..2979846f 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -264,7 +264,7 @@ class VideoDownloader(Downloader, ABC):
channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
channel = None
- if channel_name or channel_source_id:
+ if channel_name or channel_source_id or channel_url:
# Try to find the channel via info_json from yt-dlp.
try:
channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
@@ -278,7 +278,7 @@ class VideoDownloader(Downloader, ABC):
settings = download.settings or dict()
destination = settings.get('destination')
if not channel and destination:
- # Destination may override the real Channel.
+ # Destination may find Channel if not already found.
try:
channel = get_channel(directory=destination, return_dict=False)
found_channel = 'download_settings_directory'
@@ -290,7 +290,7 @@ class VideoDownloader(Downloader, ABC):
local_channel_id = settings.get('channel_id')
channel_url = settings.get('channel_url')
if not channel and (local_channel_id or channel_url):
- # Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
+ # Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
logger.info(f'Using download.settings to find channel')
try:
channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False)
@@ -320,11 +320,12 @@ class VideoDownloader(Downloader, ABC):
logger.debug(f'Downloading {url} to destination from settings')
elif channel:
out_dir = channel_directory
- logger.debug(f'Downloading {url} to channel directory')
+ logger.debug(f'Downloading {url} to channel directory: {channel_directory}')
else:
# Download to the default directory if this video has no channel.
out_dir = get_no_channel_directory()
logger.debug(f'Downloading {url} to default directory')
+
out_dir.mkdir(exist_ok=True, parents=True)
logs = None # noqa
diff --git a/modules/videos/errors.py b/modules/videos/errors.py
index 18633148..a42d741f 100644
--- a/modules/videos/errors.py
+++ b/modules/videos/errors.py
@@ -15,6 +15,12 @@ class UnknownChannel(APIError):
status_code = HTTPStatus.NOT_FOUND
+class InvalidChannel(APIError):
+ code = 'INVALID_CHANNEL'
+ message = 'The channel is not supported'
+ status_code = HTTPStatus.BAD_REQUEST
+
+
class ChannelNameConflict(APIError):
code = 'CHANNEL_NAME_CONFLICT'
message = 'The channel name is already taken.'
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 2cd84f16..265031d5 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -3,27 +3,30 @@ import dataclasses
import html
import pathlib
import re
-from typing import Tuple, Optional, Generator
+from typing import Generator
+from typing import Tuple, Optional
import pytz
from sqlalchemy import or_
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
+from modules.videos.models import Video
from wrolpi import before_startup, dates, flags
from wrolpi.captions import extract_captions
-from wrolpi.common import ConfigFile, get_media_directory, register_refresh_cleanup, limit_concurrent
+from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \
+ escape_file_name, get_media_directory
from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download
from wrolpi.errors import UnknownDirectory
from wrolpi.files.lib import split_path_stem_and_suffix
-from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
+from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
from .common import is_valid_poster, convert_image, \
generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
get_video_duration
from .errors import UnknownChannel
-from .models import Channel, Video
+from .models import Channel
logger = logger.getChild(__name__)
@@ -377,7 +380,6 @@ channel_import_logger = logger.getChild('channel_import')
@before_startup
-@register_refresh_cleanup
@limit_concurrent(1)
def import_channels_config():
"""Import channel settings to the DB. Existing channels will be updated."""
@@ -645,3 +647,41 @@ def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path
''')
results = (pathlib.Path(j['path']) for i in curs.fetchall() for j in i[0])
yield from results
+
+
+def format_videos_destination(channel_name: str = None, channel_tag: str = None, channel_url: str = None) \
+ -> pathlib.Path:
+ """Return the directory where Videos should be downloaded according to the WROLPi Config.
+
+ @warning: Directory may or may not exist."""
+ videos_destination = get_wrolpi_config().videos_destination
+
+ channel_domain = ''
+ if channel_url:
+ try:
+ channel_domain = extract_domain(channel_url)
+ except Exception as e:
+ logger.error(f'Failed to extract domain from Channel URL: {channel_url}', exc_info=e)
+ if PYTEST:
+ raise
+
+ if channel_name and not isinstance(channel_name, str):
+ raise RuntimeError('channel name must be string')
+ if channel_tag and not isinstance(channel_tag, str):
+ raise RuntimeError('channel tag must be string')
+
+ name = escape_file_name(channel_name) if channel_name else ''
+ variables = dict(
+ channel_name=name,
+ channel_tag=channel_tag or '',
+ channel_domain=channel_domain,
+ )
+
+ try:
+ videos_destination = videos_destination % variables
+ except KeyError as e:
+ raise FileNotFoundError(f'Cannot download to the defined "videos_destination"') from e
+
+ videos_destination = get_media_directory() / videos_destination.lstrip('/')
+
+ return videos_destination
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 6ae9ed9e..1b9b598e 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -1,6 +1,5 @@
import json
import pathlib
-from pathlib import Path
from typing import Optional, Dict, List
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger
@@ -11,12 +10,12 @@ from modules.videos.errors import UnknownVideo, UnknownChannel
from wrolpi.captions import read_captions
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
from wrolpi.db import get_db_curs, get_db_session, optional_session
-from wrolpi.downloader import Download, download_manager
+from wrolpi.downloader import Download, download_manager, save_downloads_config
from wrolpi.errors import InvalidDownload
-from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
+from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, move
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
-from wrolpi.tags import Tag, TagFile
+from wrolpi.tags import Tag, TagFile, save_tags_config
from wrolpi.vars import PYTEST, DELETED_VIDEO_KEYS
logger = logger.getChild(__name__)
@@ -533,10 +532,16 @@ class Channel(ModelHelper, Base):
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id')
+ tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'))
+ tag = relationship('Tag')
def __repr__(self):
return f''
+ @property
+ def tag_name(self) -> str | None:
+ return self.tag.name if self.tag else None
+
def delete_with_videos(self):
"""Delete all Video records (but not video files) related to this Channel. Then delete the Channel."""
session = Session.object_session(self)
@@ -560,12 +565,20 @@ class Channel(ModelHelper, Base):
channel_id = self.id
data = data.copy()
+ # Remove unused values.
+ data.pop('move_to_tag_directory', None)
+
# URL should not be empty string.
if 'url' in data:
data['url'] = data['url'] or None
+ session: Session = Session.object_session(self)
+
downloads = data.pop('downloads', [])
+ if tag_name := data.pop('tag_name', None):
+ self.tag = Tag.find_by_name(tag_name, session)
+
for key, value in data.items():
setattr(self, key, value)
@@ -575,8 +588,6 @@ class Channel(ModelHelper, Base):
elif isinstance(self.directory, str) and not pathlib.Path(self.directory).is_absolute():
self.directory = get_media_directory() / self.directory
- session: Session = Session.object_session(self)
-
for download in downloads:
url = None
frequency = None
@@ -608,11 +619,12 @@ class Channel(ModelHelper, Base):
config = dict(
calculate_duration=self.calculate_duration,
directory=str(self.directory),
+ downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads],
generate_posters=self.generate_posters,
name=self.name,
source_id=self.source_id,
+ tag_name=self.tag_name,
url=self.url or None,
- downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads]
)
return config
@@ -627,16 +639,18 @@ class Channel(ModelHelper, Base):
def __json__(self) -> dict:
d = dict(
- downloads=self.downloads,
directory=self.directory,
+ downloads=self.downloads,
id=self.id,
name=self.name,
+ tag_name=self.tag_name,
url=self.url,
)
return d
def dict(self, with_statistics: bool = False, with_downloads: bool = True) -> dict:
d = super(Channel, self).dict()
+ d['tag_name'] = self.tag_name
d['directory'] = \
self.directory.relative_to(get_media_directory()) if self.directory else None
if with_statistics:
@@ -706,10 +720,9 @@ class Channel(ModelHelper, Base):
"""Find a Channel with the provided id, raises an exception when no Channel is found.
@raise UnknownChannel: if the channel can not be found"""
- channel = Channel.get_by_id(id_, session=session)
- if not channel:
- raise UnknownChannel(f'Cannot find channel with id {id_}')
- return channel
+ if channel := Channel.get_by_id(id_, session=session):
+ return channel
+ raise UnknownChannel(f'Cannot find channel with id {id_}')
def get_video_entry_by_id(self, video_source_id: str) -> Optional[Dict]:
"""Search my info_json for the entry with the provided id."""
@@ -758,3 +771,40 @@ class Channel(ModelHelper, Base):
raise RuntimeError('Must provide URL to get Channel')
channel = session.query(Channel).filter_by(url=url).one_or_none()
return channel
+
+ async def move(self, directory: pathlib.Path, session: Session):
+ """Move the files of this Channel into a new directory."""
+ if not directory.is_dir():
+ raise FileNotFoundError(f'Destination directory does not exist: {directory}')
+ if self.directory and not next(self.directory.iterdir(), None):
+ raise FileExistsError(f'Refusing to move Channel when it has no files.')
+
+ old_directory = self.directory
+ self.directory = directory
+ session.flush([self, ])
+
+ # Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's
+ # directory.
+ downloads = list(self.downloads)
+ downloads.extend(list(
+ session.query(Download).filter(Download.settings.contains({'destination': str(old_directory)}))
+ ))
+ for download in downloads:
+ # Get new copy of `settings` to avoid bug where it wouldn't update.
+ settings = download.settings.copy() if download.settings else dict()
+ if settings.get('destination'):
+ download.settings = {**settings, 'destination': str(directory)}
+
+ session.commit()
+
+ # Save configs before move, this is because move imports configs.
+ from modules.videos.lib import save_channels_config
+ await save_downloads_config(session)
+ save_channels_config(session)
+ save_tags_config(session)
+
+ # Move the contents of the Channel directory into the destination directory.
+ await move(directory, *list(old_directory.iterdir()))
+
+ if old_directory.exists() and not next(old_directory.iterdir(), None):
+ old_directory.rmdir()
diff --git a/modules/videos/schema.py b/modules/videos/schema.py
index cb2b7f19..4760149d 100644
--- a/modules/videos/schema.py
+++ b/modules/videos/schema.py
@@ -37,6 +37,21 @@ class ChannelPutRequest:
self.url = self.url.strip() if self.url else None
+@dataclass
+class ChannelTagRequest:
+ tag_name: Optional[str] = None
+ directory: Optional[str] = None
+
+ def __post_init__(self):
+ if self.directory and self.directory.startswith('/'):
+ raise ValidationError('Directory must be relative to media directory')
+
+
+@dataclass
+class ChannelTagInfoRequest:
+ tag_name: Optional[str] = None
+
+
@dataclass
class ChannelDownloadRequest:
url: str
diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py
index b5aea3c8..00044b5e 100644
--- a/modules/videos/test/test_common.py
+++ b/modules/videos/test/test_common.py
@@ -332,7 +332,7 @@ def test_get_videos_directory(test_directory):
def test_get_custom_videos_directory(test_directory, test_config):
"""Custom directory can be used for videos directory."""
config = get_wrolpi_config()
- config.videos_directory = 'custom/directory/videos'
+ config.videos_destination = 'custom/directory/videos'
assert common.get_videos_directory() == (test_directory / 'custom/directory/videos')
assert (test_directory / 'custom/directory/videos').is_dir()
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index 4bd30bf1..d432591d 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -19,7 +19,7 @@ example_video_json = {
'categories': ['Howto & Style'],
'channel': 'channel name',
'channel_id': 'channel id',
- 'channel_url': 'channel url',
+ 'channel_url': 'https://example.com/example_video_json_channel',
'duration': 635,
'extractor': 'youtube',
'extractor_key': 'Youtube',
@@ -31,7 +31,7 @@ example_video_json = {
'upload_date': '20190707',
'uploader': 'uploader name',
'uploader_id': 'uploader id',
- 'uploader_url': 'uploader url',
+ 'uploader_url': 'https://example.com/example_video_json_uploader',
'view_count': 406,
'webpage_url': 'webpage url',
'webpage_url_basename': 'watch',
diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py
index fc2fee71..01907e19 100644
--- a/modules/videos/test/test_lib.py
+++ b/modules/videos/test/test_lib.py
@@ -5,8 +5,9 @@ import pytest
from modules.videos import lib
from modules.videos.downloader import ChannelDownloader, VideoDownloader
-from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
+from modules.videos.lib import parse_video_file_name, validate_video, get_statistics, format_videos_destination
from modules.videos.models import Video, Channel
+from wrolpi.common import get_wrolpi_config
from wrolpi.downloader import Download, RSSDownloader
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
@@ -20,7 +21,7 @@ from wrolpi.vars import PROJECT_DIR
('20000101foo.mp4', (None, None, None, '20000101foo')),
('something 20000101 foo.mp4', (None, None, None, 'something 20000101 foo')),
('something_20000101_foo.mp4', ('something', '20000101', None, 'foo')),
- ('foo .mp4', (None, None, None, 'foo')), # Has trailing whitespace.
+ ('foo .mp4', (None, None, None, 'foo')), # Has trailing whitespace.
('NA_20000303_vp91w5_Bob's Pancakes.mp4', (None, '20000303', 'vp91w5', 'Bob's Pancakes')),
('NA_NA_vp91w5_Bob's Pancakes.mp4', (None, None, 'vp91w5', 'Bob's Pancakes')),
('Learning Self-Reliance_20170529_p_MzsCFkUPU_Beekeeping 2017 Part 6 - Merging Hives.mp4',
@@ -216,3 +217,33 @@ def test_link_channel_and_downloads_migration(test_async_client, test_session, c
assert d2b.channel_id
assert d2c.url == 'https://example.com/channel2/video/1' and d2c.frequency is None
assert not d2c.channel_id
+
+
+@pytest.mark.asyncio
+async def test_format_videos_destination(test_async_client, test_directory):
+ """Videos destination is formatted according to the WROLPiConfig."""
+ wrolpi_config = get_wrolpi_config()
+
+ # Channel directory without a tag.
+ wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_name)s'
+ assert format_videos_destination('Simple Channel', None, None) \
+ == test_directory / 'videos/Simple Channel'
+
+ # One tag can be applied to a Channel.
+ assert format_videos_destination('Simple Channel', 'one', None) \
+ == test_directory / 'videos/one/Simple Channel'
+
+ # Channel Domain is also supported.
+ wrolpi_config.videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s'
+ assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \
+ == test_directory / 'videos/one/example.com/Simple Channel'
+
+ # Channel name is not required.
+ wrolpi_config.videos_destination = '%(channel_tag)s/%(channel_domain)s'
+ assert format_videos_destination('Simple Channel', 'one', 'https://example.com') \
+ == test_directory / 'one/example.com'
+
+ # Invalid `videos_destination` raises an error.
+ wrolpi_config.videos_destination = '%(channel_tag)s/%(channel)s'
+ with pytest.raises(FileNotFoundError):
+ assert format_videos_destination()
diff --git a/modules/videos/test/test_models.py b/modules/videos/test/test_models.py
index 4a678064..436d4cb6 100644
--- a/modules/videos/test/test_models.py
+++ b/modules/videos/test/test_models.py
@@ -1,5 +1,10 @@
+import pathlib
+
import pytest
+from modules.videos.models import Video, Channel
+from wrolpi.common import get_relative_to_media_directory, get_wrolpi_config
+
@pytest.mark.asyncio()
async def test_ffprobe_stream_methods(simple_video):
@@ -13,3 +18,24 @@ async def test_ffprobe_stream_methods(simple_video):
assert simple_video.get_streams_by_codec_name('h264')
assert simple_video.get_streams_by_codec_name('aac')
assert simple_video.get_streams_by_codec_name('mov_text')
+
+
+@pytest.mark.asyncio
+async def test_channel_move(test_async_client, test_session, test_directory, channel_factory, video_factory):
+ """A Channel can be moved to another directory, any files in the Channel's directory are moved."""
+ channel = channel_factory(name='Channel Name')
+ video = video_factory(title='Vid', channel_id=channel.id)
+ test_session.commit()
+ assert video.channel_id == channel.id
+ assert str(get_relative_to_media_directory(video.video_path)) == 'videos/Channel Name/Vid.mp4'
+
+ foo = test_directory / 'foo/New Channel Directory'
+ foo.mkdir(parents=True)
+ await channel.move(foo, test_session)
+ assert channel.directory == foo
+ assert channel.directory != (test_directory / 'New Channel Directory')
+ video = test_session.query(Video).one()
+ assert str(get_relative_to_media_directory(video.video_path)) == 'foo/New Channel Directory/Vid.mp4'
+ assert str(video.video_path).startswith(str(channel.directory))
+ # Old directory is deleted.
+ assert not (test_directory / 'Channel Name').exists()
diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py
index 89ec00bf..79775c1e 100644
--- a/modules/videos/video/lib.py
+++ b/modules/videos/video/lib.py
@@ -85,7 +85,7 @@ def search_videos(
joins = list()
join_video = False
- params = dict(search_str=search_str, offset=offset)
+ params = dict(search_str=search_str, offset=offset or 0)
if channel_id:
wheres.append('v.channel_id = %(channel_id)s')
joins.append('LEFT JOIN channel c ON c.id = v.channel_id')
diff --git a/modules/zim/lib.py b/modules/zim/lib.py
index 7d4140c3..4e9a3936 100644
--- a/modules/zim/lib.py
+++ b/modules/zim/lib.py
@@ -464,7 +464,7 @@ def zim_download_url_to_name(url: str) -> Tuple[str, str]:
def get_zim_directory() -> pathlib.Path:
- zim_directory = get_media_directory() / get_wrolpi_config().zims_directory
+ zim_directory = get_media_directory() / get_wrolpi_config().zims_destination
return zim_directory
diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py
index 828ff465..7899f9fd 100644
--- a/modules/zim/test/test_lib.py
+++ b/modules/zim/test/test_lib.py
@@ -385,6 +385,6 @@ def test_get_custom_zims_directory(test_directory, test_config):
"""Custom directory can be used for zims directory."""
assert lib.get_zim_directory() == (test_directory / 'zims')
- get_wrolpi_config().zims_directory = 'custom/zims'
+ get_wrolpi_config().zims_destination = 'custom/zims'
assert lib.get_zim_directory() == (test_directory / 'custom/zims')
diff --git a/requirements.txt b/requirements.txt
index 0b276215..f18abcf8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -27,4 +27,4 @@ srt==3.5.3
vininfo[cli]==1.8.0
websockets==12.0
webvtt-py==0.4.6
-yt-dlp==2024.8.1
\ No newline at end of file
+yt-dlp==2024.8.6
\ No newline at end of file
diff --git a/scripts/singlefile_to_archive.py b/scripts/singlefile_to_archive.py
index b3bd5bd1..227e46fc 100755
--- a/scripts/singlefile_to_archive.py
+++ b/scripts/singlefile_to_archive.py
@@ -10,7 +10,7 @@ import tempfile
from modules.archive import lib as archive_lib
from wrolpi.cmd import READABILITY_BIN
-from wrolpi.common import html_screenshot, format_html_string, logger
+from wrolpi.common import html_screenshot, format_html_string, logger, format_html_file
logger = logger.getChild('singlefile_to_archive')
@@ -31,6 +31,8 @@ def singlefile_to_archive_files(singlefile: pathlib.Path):
"""Extract Readability files and Screenshot file next to the provided Singlefile."""
contents = singlefile.read_bytes()
+ format_html_file(singlefile)
+
readability_html_path = singlefile.with_suffix('.readability.html')
readability_json_path = singlefile.with_suffix('.readability.json')
readability_text_path = singlefile.with_suffix('.readability.txt')
diff --git a/wrolpi/common.py b/wrolpi/common.py
index 88f6b45e..2ac12c31 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -276,7 +276,10 @@ class ModelHelper:
def flush(self):
"""A convenience function which flushes this record using its DB Session."""
- Session.object_session(self).flush([self])
+ if session := Session.object_session(self):
+ session.flush([self])
+ else:
+ raise RuntimeError(f'{self} is not in a session!')
class tsvector(types.TypeDecorator):
@@ -370,7 +373,8 @@ class ConfigFile:
if config_file.is_file():
# Use the config file to get the values the user set.
with config_file.open('rt') as fh:
- self._config.update(yaml.load(fh, Loader=yaml.Loader))
+ config_data = yaml.load(fh, Loader=yaml.Loader)
+ self._config.update(config_data)
return self
def _get_backup_filename(self):
@@ -467,7 +471,7 @@ class ConfigFile:
class WROLPiConfig(ConfigFile):
file_name = 'wrolpi.yaml'
default_config = dict(
- archive_directory='archive',
+ archive_destination='archive/%(domain)s',
download_on_startup=True,
download_timeout=0,
hotspot_device='wlan0',
@@ -476,12 +480,12 @@ class WROLPiConfig(ConfigFile):
hotspot_ssid='WROLPi',
ignore_outdated_zims=False,
ignored_directories=['config', 'tags'],
- map_directory='map',
+ map_destination='map',
nav_color='violet',
throttle_on_startup=False,
- videos_directory='videos',
+ videos_destination='videos/%(channel_tag)s/%(channel_name)s',
wrol_mode=False,
- zims_directory='zims',
+ zims_destination='zims',
)
def get_file(self) -> Path:
@@ -586,28 +590,36 @@ class WROLPiConfig(ConfigFile):
self.update({'ignored_directories': value})
@property
- def videos_directory(self) -> str:
- return self._config['videos_directory']
+ def videos_destination(self) -> str:
+ return self._config['videos_destination']
- @videos_directory.setter
- def videos_directory(self, value: str):
- self.update({'videos_directory': value})
+ @videos_destination.setter
+ def videos_destination(self, value: str):
+ self.update({'videos_destination': value})
@property
- def archive_directory(self) -> str:
- return self._config['archive_directory']
+ def archive_destination(self) -> str:
+ return self._config['archive_destination']
- @archive_directory.setter
- def archive_directory(self, value: str):
- self.update({'archive_directory': value})
+ @archive_destination.setter
+ def archive_destination(self, value: str):
+ self.update({'archive_destination': value})
@property
- def map_directory(self) -> str:
- return self._config['map_directory']
+ def map_destination(self) -> str:
+ return self._config['map_destination']
- @map_directory.setter
- def map_directory(self, value: str):
- self.update({'map_directory': value})
+ @map_destination.setter
+ def map_destination(self, value: str):
+ self.update({'map_destination': value})
+
+ @property
+ def zims_destination(self) -> str:
+ return self._config['zims_destination']
+
+ @zims_destination.setter
+ def zims_destination(self, value: str):
+ self.update({'zims_destination': value})
@property
def nav_color(self) -> str:
@@ -617,14 +629,6 @@ class WROLPiConfig(ConfigFile):
def nav_color(self, value: str):
self.update({'nav_color': value})
- @property
- def zims_directory(self) -> str:
- return self._config['zims_directory']
-
- @zims_directory.setter
- def zims_directory(self, value: str):
- self.update({'zims_directory': value})
-
WROLPI_CONFIG: WROLPiConfig = WROLPiConfig()
TEST_WROLPI_CONFIG: WROLPiConfig = None
diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py
index a0027892..2492e107 100644
--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@@ -36,6 +36,7 @@ import wrolpi.root_api # noqa
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode
+from wrolpi.common import logger
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
from wrolpi.dates import set_test_now
@@ -47,6 +48,8 @@ from wrolpi.files.models import Directory, FileGroup
from wrolpi.tags import Tag, upsert_tag
from wrolpi.vars import PROJECT_DIR
+logger = logger.getChild(__name__)
+
def get_test_db_engine():
suffix = str(uuid1()).replace('-', '')
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index ee12fab0..c0636a23 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -27,7 +27,7 @@ from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import pid_is_running
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \
- limit_concurrent, wrol_mode_enabled
+ limit_concurrent, wrol_mode_enabled, background_task
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload
@@ -226,7 +226,7 @@ class Download(ModelHelper, Base): # noqa
from modules.videos.lib import save_channels_config
save_channels_config()
# Save download config again because this download is now removed from the download lists.
- api_app.add_task(save_downloads_config())
+ background_task(save_downloads_config())
@staticmethod
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
@@ -538,9 +538,9 @@ class DownloadManager:
session.flush(downloads)
try:
# Start downloading ASAP.
- api_app.add_task(self.dispatch_downloads())
+ background_task(self.dispatch_downloads())
# Save the config now that new Downloads exist.
- api_app.add_task(save_downloads_config())
+ background_task(save_downloads_config())
except RuntimeError:
# Event loop isn't running. Probably testing?
if not PYTEST:
@@ -574,9 +574,6 @@ class DownloadManager:
from modules.videos.models import Channel
if channel := Channel.get_by_url(url=download.url, session=session):
download.channel_id = channel.id
- if destination := (download.settings or dict()).get('destination'):
- if channel := Channel.get_by_path(destination, session):
- download.channel_id = channel.id
session.commit()
@@ -668,7 +665,7 @@ class DownloadManager:
with get_db_curs(commit=True) as curs:
curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
- api_app.add_task(save_downloads_config())
+ background_task(save_downloads_config())
DOWNLOAD_SORT = (
DownloadStatus.pending,
@@ -742,7 +739,7 @@ class DownloadManager:
session.commit()
# Save the config now that some Downloads renewed.
- api_app.add_task(save_downloads_config())
+ background_task(save_downloads_config())
@staticmethod
def get_downloads(session: Session) -> List[Download]:
@@ -1025,7 +1022,7 @@ class DownloadManager:
stmt = self._delete_downloads_q(once=True)
deleted_ids = [i for i, in session.execute(stmt).fetchall()]
session.commit()
- api_app.add_task(save_downloads_config())
+ background_task(save_downloads_config())
return deleted_ids
@staticmethod
@@ -1130,7 +1127,7 @@ async def signal_download_download(download_id: int, download_url: str):
# Allow the download to resume.
download_manager.unkill_download(download_id)
# Save the config now that the Download has finished.
- api_app.add_task(save_downloads_config())
+ background_task(save_downloads_config())
except asyncio.CancelledError as e:
worker_logger.warning('Canceled!', exc_info=e)
return
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 09282acc..e0ffb19b 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -137,7 +137,7 @@ def get_settings(_: Request):
ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories]
settings = {
- 'archive_directory': config.archive_directory,
+ 'archive_directory': config.archive_destination,
'download_manager_disabled': download_manager.is_disabled,
'download_manager_stopped': download_manager.is_stopped,
'download_on_startup': config.download_on_startup,
@@ -150,15 +150,15 @@ def get_settings(_: Request):
'ignore_outdated_zims': config.ignore_outdated_zims,
'ignored_directories': ignored_directories,
'log_level': api_app.shared_ctx.log_level.value,
- 'map_directory': config.map_directory,
+ 'map_directory': config.map_destination,
'nav_color': config.nav_color,
'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative.
'throttle_on_startup': config.throttle_on_startup,
'throttle_status': admin.throttle_status().name,
'version': __version__,
- 'videos_directory': config.videos_directory,
+ 'videos_directory': config.videos_destination,
'wrol_mode': config.wrol_mode,
- 'zims_directory': config.zims_directory,
+ 'zims_directory': config.zims_destination,
}
return json_response(settings)
@@ -190,25 +190,25 @@ async def update_settings(_: Request, body: schema.SettingsRequest):
if not config:
raise InvalidConfig()
- if body.archive_directory and pathlib.Path(body.archive_directory).is_absolute():
+ if body.archive_destination and pathlib.Path(body.archive_destination).is_absolute():
raise InvalidConfig('Archive directory must be relative to media directory')
- elif not body.archive_directory:
- config['archive_directory'] = wrolpi_config.default_config['archive_directory']
+ elif not body.archive_destination:
+ config['archive_destination'] = wrolpi_config.default_config['archive_destination']
- if body.videos_directory and pathlib.Path(body.videos_directory).is_absolute():
+ if body.videos_destination and pathlib.Path(body.videos_destination).is_absolute():
raise InvalidConfig('Videos directory must be relative to media directory')
- elif not body.videos_directory:
- config['videos_directory'] = wrolpi_config.default_config['videos_directory']
+ elif not body.videos_destination:
+ config['videos_destination'] = wrolpi_config.default_config['videos_destination']
- if body.map_directory and pathlib.Path(body.map_directory).is_absolute():
+ if body.map_destination and pathlib.Path(body.map_destination).is_absolute():
raise InvalidConfig('Map directory must be relative to media directory')
- elif not body.map_directory:
- config['map_directory'] = wrolpi_config.default_config['map_directory']
+ elif not body.map_destination:
+ config['map_destination'] = wrolpi_config.default_config['map_destination']
- if body.zims_directory and pathlib.Path(body.zims_directory).is_absolute():
+ if body.zims_destination and pathlib.Path(body.zims_destination).is_absolute():
raise InvalidConfig('Zims directory must be relative to media directory')
- elif not body.zims_directory:
- config['zims_directory'] = wrolpi_config.default_config['zims_directory']
+ elif not body.zims_destination:
+ config['zims_destination'] = wrolpi_config.default_config['zims_destination']
log_level = config.pop('log_level', None)
if isinstance(log_level, int):
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index f7bc6aed..2fadb3c5 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -32,7 +32,7 @@ class SemanticUIColors(enum.StrEnum):
@dataclass
class SettingsRequest:
- archive_directory: Optional[str] = None
+ archive_destination: Optional[str] = None
download_on_startup: Optional[bool] = None
download_timeout: Optional[int] = None
hotspot_device: Optional[str] = None
@@ -42,14 +42,14 @@ class SettingsRequest:
hotspot_status: Optional[bool] = None
ignore_outdated_zims: Optional[bool] = None
log_level: Optional[int] = None
- map_directory: Optional[str] = None
+ map_destination: Optional[str] = None
nav_color: Optional[str] = None
media_directory: Optional[str] = None
throttle_on: Optional[bool] = None
throttle_on_startup: Optional[bool] = None
- videos_directory: Optional[str] = None
+ videos_destination: Optional[str] = None
wrol_mode: Optional[bool] = None
- zims_directory: Optional[str] = None
+ zims_destination: Optional[str] = None
def __post_init__(self):
self.nav_color = self.nav_color.lower() if self.nav_color else None
diff --git a/wrolpi/tags.py b/wrolpi/tags.py
index d55e324a..14bd63eb 100644
--- a/wrolpi/tags.py
+++ b/wrolpi/tags.py
@@ -426,7 +426,6 @@ def delete_tag(tag_id: int, session: Session = None):
sync_tags_directory()
-@register_refresh_cleanup
@optional_session
def import_tags_config(session: Session = None):
"""Reads the Tags and TagFiles from the config file, upserts them in the DB."""
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
index 76c7e0d2..91c0aa82 100644
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@@ -530,14 +530,15 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
request, response = await test_async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
if expected_channels:
- assert response.json['channels'] == expected_channels
+ for channel, expected_channel in zip_longest(response.json['channels'], expected_channels):
+ assert_dict_contains(channel, expected_channel)
assert response.json['domains'] == expected_domains
await assert_results(
dict(search_str='foo'),
[
- {'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []},
- {'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []},
+ {'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []},
+ {'directory': 'videos/Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []},
],
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}],
)
@@ -546,14 +547,14 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
await assert_results(
dict(search_str='foo l'),
[
- {'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}],
+ {'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}],
[],
)
await assert_results(
dict(search_str='bar'),
[
- {'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []}
+ {'directory': 'videos/Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []}
],
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}],
)
@@ -705,29 +706,28 @@ def test_recursive_errors():
@pytest.mark.asyncio
async def test_settings_special_directories(test_async_client, test_config):
"""Maintainer can change special directories."""
- assert get_wrolpi_config().archive_directory == 'archive'
- data = {'archive_directory': 'archives'}
+ data = {'archive_destination': 'archives'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
- assert get_wrolpi_config().archive_directory == 'archives'
+ assert get_wrolpi_config().archive_destination == 'archives'
- data = {'archive_directory': '/absolute/not/allowed'}
+ data = {'archive_destination': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
- data = {'videos_directory': '/absolute/not/allowed'}
+ data = {'videos_destination': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
- data = {'map_directory': '/absolute/not/allowed'}
+ data = {'map_destination': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
- data = {'zims_directory': '/absolute/not/allowed'}
+ data = {'zims_destination': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
# Empty directory restores default.
- data = {'archive_directory': ''}
+ data = {'archive_destination': ''}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
- assert get_wrolpi_config().archive_directory == 'archive'
+ assert get_wrolpi_config().archive_destination == 'archive/%(domain)s'