From 1ebcae0965601a37a7d4c2a0fd9a04141ea0de56 Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 22 Dec 2025 17:56:35 -0700 Subject: [PATCH] Allowing user to configure collection download destination directories, as well as directories in the file structure. This will allow sorting archives by destination year, for example. --- app/src/api.js | 13 ++ app/src/components/Archive.js | 133 ++++++++++--- app/src/components/Common.js | 5 +- app/src/components/Videos.js | 28 ++- app/src/hooks/customHooks.js | 2 +- main.py | 20 +- modules/archive/__init__.py | 9 +- modules/archive/api.py | 32 ++- modules/archive/lib.py | 204 +++++++++++++++----- modules/archive/schema.py | 5 + modules/archive/test/test_api.py | 39 ++++ modules/archive/test/test_domain_tagging.py | 51 +---- modules/archive/test/test_lib.py | 126 ++++++++++-- modules/videos/lib.py | 4 + wrolpi/collections/models.py | 94 +++++++-- wrolpi/collections/test/test_api.py | 28 ++- wrolpi/collections/test/test_models.py | 107 ++++++++++ wrolpi/common.py | 20 +- wrolpi/contexts.py | 8 + wrolpi/files/test/test_lib.py | 3 +- wrolpi/test/test_common.py | 9 + 21 files changed, 780 insertions(+), 160 deletions(-) diff --git a/app/src/api.js b/app/src/api.js index fa8fbdce..44e4091c 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -429,6 +429,19 @@ export async function updateVideoDownloaderConfig(config) { return await updateConfig('videos_downloader.yaml', config); } +export async function fetchArchiveDownloaderConfig() { + return await getConfig('archives_downloader.yaml'); +} + +export async function updateArchiveDownloaderConfig(config) { + return await updateConfig('archives_downloader.yaml', config); +} + +export async function postArchiveFileFormat(archive_file_format) { + const body = {archive_file_format}; + return await apiPost(`${ARCHIVES_API}/file_format`, body); +} + export async function fetchBrowserProfiles() { const response = await apiGet(`${API_URI}/videos/browser-profiles`); if (response.status === 200) { diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index e5c822f8..de8b20ea 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -43,14 +43,18 @@ import { import { deleteArchives, deleteDomain, + fetchArchiveDownloaderConfig, generateArchiveScreenshot, getCollectionTagInfo, + postArchiveFileFormat, postDownload, refreshDomain, tagDomain, tagFileGroup, - untagFileGroup + untagFileGroup, + updateArchiveDownloaderConfig, } from "../api"; +import {InputForm, useForm} from "../hooks/useForm"; import {CollectionTagModal} from "./collections/CollectionTagModal"; import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; @@ -658,9 +662,67 @@ export function DomainEditPage() { ; } +function ArchiveFileNameForm({form}) { + const [message, setMessage] = React.useState(null); + + const onChange = async (value) => { + const response = await postArchiveFileFormat(value); + const {error, preview} = await response.json(); + if (error) { + setMessage({content: error, header: 'Invalid File Name', negative: true}); + } else { + setMessage({content: preview, header: 'File Name Preview', positive: true}); + } + } + + const label = +

Variables:

+ +

Subdirectories supported: %(download_year)s/%(title)s

+ } + />; + + return +} + function ArchiveSettingsPage() { + useTitle('Archive Settings'); + const {t} = React.useContext(ThemeContext); + const emptyFormData = { + file_name_format: '%(download_datetime)s_%(title)s', + }; + + const configSubmitter = async () => { + return await updateArchiveDownloaderConfig(configForm.formData); + }; + + const configForm = useForm({ + fetcher: fetchArchiveDownloaderConfig, + submitter: configSubmitter, + emptyFormData, + }); + const urlClipboardButton = navigator.clipboard.writeText(API_ARCHIVE_UPLOAD_URI)} @@ -675,29 +737,56 @@ function ArchiveSettingsPage() { />; return -
SingleFile Browser Extension
+ +
Archive Downloader Config
-

- These are the settings necessary to configure the SingleFile Browser - Extension to automatically upload to your WROLPi. -

+
+ + + + + + + + + Save + + + +
+ - - - - - - + +
SingleFile Browser Extension
+ +

+ These are the settings necessary to configure the SingleFile Browser + Extension to automatically upload to your WROLPi. +

+ + + + + + + + } diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 63d577e4..0b96b71e 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -759,6 +759,7 @@ export function InfoPopup({ iconSize = null, header = '', iconStyle = {marginLeft: '0.25em', marginRight: '0.25em', marginTop: '-0.5em'}, + ...props }) { const trigger = ; return } @@ -778,6 +780,7 @@ export function InfoHeader({ headerContent, popupContent, popupPosition = null, + popupProps = {}, for_ = null, required = false, ...props @@ -787,7 +790,7 @@ export function InfoHeader({
{headerContent} {required && }
- + } diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 8d2929e4..e8380ad0 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -244,17 +244,37 @@ function VideoFileNameForm({form}) { } } + const label = +

Common yt-dlp variables:

+
    +
  • %(title)s - Video title
    • +
  • %(uploader)s - Channel/uploader name
    • +
  • %(upload_date)s - Upload date (YYYYMMDD)
    • +
  • %(id)s - Video ID
    • +
  • %(ext)s - File extension
    • +
  • %(channel)s - Channel name
    • +
  • %(duration)s - Duration in seconds
    • +
  • %(playlist_index)s - Playlist index
    • +
+

See yt-dlp docs for all variables. Subdirectories supported.

+ } + />; + return } -function VideosSettings() { +function VideosSettingsPage() { useTitle('Videos Settings'); const emptyFormData = { @@ -301,7 +321,7 @@ function VideosSettings() { if (_.isEmpty(tempProfiles)) { tempProfiles = emptyProfilesOptions; } - console.debug('VideosSettings: Got browser profiles:', tempProfiles); + console.debug('VideosSettingsPage: Got browser profiles:', tempProfiles); setBrowserProfilesOptions(tempProfiles); }; @@ -523,7 +543,7 @@ export function VideosRoute(props) { }/> }/> - }/> + }/> }/> }/> }/> diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index c526c977..9cacb7bb 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -353,7 +353,7 @@ export const useSearchArchives = (defaultLimit) => { useEffect(() => { localSearchArchives(); - }, [searchStr, limit, domain, order, activePage, JSON.stringify(activeTags), headline]); + }, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), headline]); const setSearchStr = (value) => { updateQuery({q: value, o: 0, order: undefined}); diff --git a/main.py b/main.py index 9bb082fc..1ba51edd 100755 --- a/main.py +++ b/main.py @@ -215,9 +215,19 @@ async def initialize_configs(app: Sanic): async def handle_server_shutdown(*args, **kwargs): """Stop downloads when server is shutting down.""" logger.warning('Shutting down') - download_manager.stop() - await cancel_refresh_tasks() - await cancel_background_tasks() + try: + download_manager.stop() + except AttributeError: + # Ignore AttributeError when context is empty, which can happen during server shutdown + pass + try: + await cancel_refresh_tasks() + except Exception as e: + logger.error('cancel_refresh_tasks failed. This is probably fine.', exc_info=e) + try: + await cancel_background_tasks() + except Exception as e: + logger.error('cancel_background_tasks failed. This is probably fine', exc_info=e) @api_app.signal(Event.SERVER_SHUTDOWN_AFTER) @@ -264,6 +274,10 @@ async def start_single_tasks(app: Sanic): with log_and_suppress(Exception, message='Failed to import domains config'): import_domains_config() logger.debug('domains config imported') + with log_and_suppress(Exception, message='Failed to import archive downloader config'): + from modules.archive.lib import get_archive_downloader_config + get_archive_downloader_config().import_config() + logger.debug('archive downloader config imported') with log_and_suppress(Exception, message='Failed to import inventories config'): import_inventories_config() logger.debug('inventories config imported') diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 56745684..b64efd55 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -49,12 +49,13 @@ class ArchiveDownloader(Downloader, ABC): if download.settings and download.settings.get('destination'): destination = pathlib.Path(download.settings['destination']) - # If no destination provided, look up the domain collection's directory + # If no destination provided, use the domain collection's directory + # get_or_set_directory will format from template and save on first use if not destination: - with get_db_session() as session: + with get_db_session(commit=True) as session: collection = lib.get_or_create_domain_collection(session, download.url) - if collection and collection.directory: - destination = collection.directory + if collection: + destination = collection.get_or_set_directory(session) if DOCKERIZED or PYTEST: # Perform the archive in the Archive docker container. (Typically in the development environment). diff --git a/modules/archive/api.py b/modules/archive/api.py index cc59594c..6cd9a876 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -70,6 +70,26 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest): return json_response(ret) +@archive_bp.post('/file_format') +@openapi.definition( + description='Preview the archive file format', + body=schema.ArchiveFileFormatRequest, +) +@validate(schema.ArchiveFileFormatRequest) +async def post_file_format(_: Request, body: schema.ArchiveFileFormatRequest): + try: + preview = lib.preview_archive_filename(body.archive_file_format) + return json_response(dict( + archive_file_format=body.archive_file_format, + preview=preview, + )) + except RuntimeError as e: + return json_response(dict( + error=str(e), + archive_file_format=body.archive_file_format, + ), status=HTTPStatus.BAD_REQUEST) + + @archive_bp.get('/upload') @openapi.definition( summary='A message to confirm to the user that they have the correct upload URL.' @@ -106,7 +126,17 @@ async def singlefile_upload_switch_handler(url=None): logger.info(f'singlefile_upload_switch_handler queue size: {q_size}') try: - archive = await lib.singlefile_to_archive(singlefile) + # Get URL from singlefile to determine destination + url = lib.get_url_from_singlefile(singlefile) + + # Get/create domain collection and get/set its directory + destination = None + with get_db_session(commit=True) as session: + collection = lib.get_or_create_domain_collection(session, url) + if collection: + destination = collection.get_or_set_directory(session) + + archive = await lib.singlefile_to_archive(singlefile, destination=destination) except Exception as e: logger.error(f'singlefile_upload_switch_handler failed', exc_info=e) Events.send_upload_archive_failed(f'Failed to convert singlefile to archive: {e}') diff --git a/modules/archive/lib.py b/modules/archive/lib.py index ce9fd4ce..8ced1992 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -213,6 +213,143 @@ def import_domains_config(): logger.info('Importing domains config completed') +@dataclass +class ArchiveDownloaderConfigValidator: + """Validator for archives_downloader.yaml config.""" + file_name_format: str + version: int = 0 + + def __post_init__(self): + # Validate file_name_format contains required %(title)s variable + if '%(title)s' not in self.file_name_format: + raise ValueError('file_name_format must contain %(title)s') + + +class ArchiveDownloaderConfig(ConfigFile): + """ + Config file for Archive Downloader settings. + + This config mirrors videos_downloader.yaml and holds settings for how + archives are downloaded and named. + + Format: + version: 0 + file_name_format: '%(download_date)s_%(title)s' + + Variables available in file_name_format: + - %(title)s - Page title (extracted from HTML) + - %(download_datetime)s - Full download datetime (YYYY-MM-DD-HH-MM-SS) + - %(download_date)s - Download date only (YYYY-MM-DD) + - %(download_year)s - Download year + - %(download_month)s - Download month (zero-padded) + - %(download_day)s - Download day (zero-padded) + - %(domain)s - Domain name + + Subdirectories can be included in the format, e.g.: + '%(download_year)s/%(download_datetime)s_%(title)s' + """ + file_name = 'archives_downloader.yaml' + validator = ArchiveDownloaderConfigValidator + default_config = dict( + version=0, + file_name_format='%(download_datetime)s_%(title)s', + ) + + @property + def file_name_format(self) -> str: + return self._config['file_name_format'] + + +# Global instance for archive downloader config +ARCHIVE_DOWNLOADER_CONFIG: ArchiveDownloaderConfig = ArchiveDownloaderConfig() + + +def get_archive_downloader_config() -> ArchiveDownloaderConfig: + """Get the global archive downloader config instance.""" + return ARCHIVE_DOWNLOADER_CONFIG + + +def format_archive_filename( + title: str, + domain: str = None, + download_date: datetime = None, +) -> str: + """Format archive filename using config template. + + This function formats the filename (and optional subdirectory path) for an archive + using the file_name_format from archives_downloader.yaml. + + Args: + title: Page title (extracted from HTML) + domain: Domain name of the archived URL + download_date: Date of download (defaults to now) + + Returns: + Formatted filename/path (without .html extension, added later) + + Example: + format_archive_filename("My Article", "example.com") + # With default format: "2025-12-22_My Article" + # With "%(download_year)s/%(title)s": "2025/My Article" + """ + config = get_archive_downloader_config() + template = config.file_name_format + + download_date = download_date or now() + + variables = dict( + title=escape_file_name(title) if title else 'untitled', + domain=domain or '', + download_datetime=archive_strftime(download_date), + download_date=download_date.strftime('%Y-%m-%d'), + download_year=str(download_date.year), + download_month=f'{download_date.month:02d}', + download_day=f'{download_date.day:02d}', + ) + + try: + return template % variables + except KeyError as e: + logger.error(f'Invalid variable in archive file_name_format: {e}') + # Fallback to default format + return f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}' + + +def preview_archive_filename(file_name_format: str) -> str: + """Preview the archive filename using sample data. + + Args: + file_name_format: The format template to preview + + Returns: + A preview string showing what the filename would look like + + Raises: + RuntimeError: If the format is invalid or missing required variables + """ + if '%(title)s' not in file_name_format: + raise RuntimeError('file_name_format must contain %(title)s') + + sample_date = now() + variables = dict( + title='Example Page Title', + domain='example.com', + download_datetime=archive_strftime(sample_date), + download_date=sample_date.strftime('%Y-%m-%d'), + download_year=str(sample_date.year), + download_month=f'{sample_date.month:02d}', + download_day=f'{sample_date.day:02d}', + ) + + try: + preview = file_name_format % variables + return f'{preview}.html' + except KeyError as e: + raise RuntimeError(f'Invalid variable: {e}') + except ValueError as e: + raise RuntimeError(f'Invalid format: {e}') + + @dataclass class ArchiveFiles: """Every Archive will have some of these files.""" @@ -241,38 +378,6 @@ def get_archive_directory() -> pathlib.Path: return archive_directory -def get_domain_directory(url: str) -> pathlib.Path: - """Get the archive directory for a particular domain.""" - domain = extract_domain(url) - directory = get_archive_directory() / domain - if directory.is_dir(): - return directory - elif directory.exists(): - raise FileNotFoundError(f'Domain directory {directory} is already a file') - - directory.mkdir(parents=True, exist_ok=True) - return directory - - -def get_archive_destination(domain_collection: 'Collection') -> pathlib.Path: - """ - Get the destination directory for archives in a domain collection. - - Args: - domain_collection: The domain Collection (kind='domain') - - Returns: - Path where archives should be placed. If collection has a directory, - returns that directory. Otherwise returns the default archives// path. - """ - if domain_collection.directory: - # Restricted collection - use collection directory - return get_media_directory() / domain_collection.directory - else: - # Unrestricted collection - use default archive// path - return get_archive_directory() / domain_collection.name - - # File names include domain and datetime. MAXIMUM_ARCHIVE_FILE_CHARACTER_LENGTH = 200 @@ -285,22 +390,35 @@ def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.P title: Title for the archive files destination: Optional directory to save files to. If provided, files go here instead of archive/ """ + domain = extract_domain(url) if destination: directory = destination - directory.mkdir(parents=True, exist_ok=True) else: - directory = get_domain_directory(url) - # Datetime is valid in Linux and Windows. - dt = archive_strftime(now()) + # Fallback to archive// when no destination provided + directory = get_archive_directory() / domain + directory.mkdir(parents=True, exist_ok=True) - title = escape_file_name(title or 'NA') + # Use the configured file format template + title = title or 'NA' title = title[:MAXIMUM_ARCHIVE_FILE_CHARACTER_LENGTH] - prefix = f'{dt}_{title}' - singlefile_path = directory / f'{prefix}.html' - readability_path = directory / f'{prefix}.readability.html' - readability_txt_path = directory / f'{prefix}.readability.txt' - readability_json_path = directory / f'{prefix}.readability.json' - screenshot_path = directory / f'{prefix}.png' + prefix = format_archive_filename(title, domain=domain) + + # Handle subdirectories in the format (e.g., "%(download_year)s/%(title)s") + if '/' in prefix: + # Prefix includes subdirectories, create them relative to base directory + full_path = directory / prefix + full_path.parent.mkdir(parents=True, exist_ok=True) + singlefile_path = full_path.with_suffix('.html') + readability_path = full_path.parent / f'{full_path.name}.readability.html' + readability_txt_path = full_path.parent / f'{full_path.name}.readability.txt' + readability_json_path = full_path.parent / f'{full_path.name}.readability.json' + screenshot_path = full_path.with_suffix('.png') + else: + singlefile_path = directory / f'{prefix}.html' + readability_path = directory / f'{prefix}.readability.html' + readability_txt_path = directory / f'{prefix}.readability.txt' + readability_json_path = directory / f'{prefix}.readability.json' + screenshot_path = directory / f'{prefix}.png' paths = (singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path) diff --git a/modules/archive/schema.py b/modules/archive/schema.py index f3cfc728..dde5853d 100644 --- a/modules/archive/schema.py +++ b/modules/archive/schema.py @@ -30,3 +30,8 @@ class ArchiveSearchRequest: class ArchiveSearchResponse: file_groups: List[ArchiveDict] totals: dict + + +@dataclass +class ArchiveFileFormatRequest: + archive_file_format: str diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index dc233e55..03701c6b 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -406,3 +406,42 @@ async def test_archive_generate_screenshot(test_session, async_client, archive_f request, response = await async_client.post(f'/api/archive/{archive_wrol.id}/generate_screenshot') assert response.status_code == HTTPStatus.FORBIDDEN await wrol_mode_fixture(False) + + +@pytest.mark.asyncio +async def test_archive_file_format_preview(async_client): + """Test the archive file format preview endpoint.""" + # Test valid format with download_datetime (default format) + data = {'archive_file_format': '%(download_datetime)s_%(title)s'} + request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data)) + assert response.status_code == HTTPStatus.OK + assert 'preview' in response.json + assert response.json['preview'].endswith('.html') + assert 'Example Page Title' in response.json['preview'] + # Verify datetime format YYYY-MM-DD-HH-MM-SS + import re + assert re.search(r'\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}', response.json['preview']) + + # Test valid format with download_date (date only) + data = {'archive_file_format': '%(download_date)s_%(title)s'} + request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data)) + assert response.status_code == HTTPStatus.OK + assert re.search(r'\d{4}-\d{2}-\d{2}_', response.json['preview']) + + # Test format with year subdirectory + data = {'archive_file_format': '%(download_year)s/%(download_datetime)s_%(title)s'} + request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data)) + assert response.status_code == HTTPStatus.OK + assert '/' in response.json['preview'] # Contains subdirectory + + # Test invalid format - missing title + data = {'archive_file_format': '%(download_date)s'} + request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert 'error' in response.json + + # Test invalid format - unknown variable + data = {'archive_file_format': '%(invalid_var)s_%(title)s'} + request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert 'error' in response.json diff --git a/modules/archive/test/test_domain_tagging.py b/modules/archive/test/test_domain_tagging.py index abfd22dd..d2d4124f 100644 --- a/modules/archive/test/test_domain_tagging.py +++ b/modules/archive/test/test_domain_tagging.py @@ -70,10 +70,15 @@ async def test_tag_unrestricted_domain_collection(async_client, test_session, ta @pytest.mark.asyncio async def test_tag_domain_collection_moves_files( - test_session, archive_factory, tag_factory, test_directory, make_files_structure, archive_directory, - await_switches, + async_client, test_session, archive_factory, tag_factory, test_directory, make_files_structure, + archive_directory, await_switches, test_wrolpi_config, ): """Tagging domain collection with directory moves archive files.""" + # Configure template to include tag in path + from wrolpi.common import get_wrolpi_config + config = get_wrolpi_config() + config.update({'archive_destination': 'archive/%(domain_tag)s/%(domain)s'}) + # Create domain collection with directory # Use archive_directory fixture to match where archive_factory creates files domain_dir = archive_directory / 'test.com' @@ -254,48 +259,6 @@ async def test_get_or_create_domain_collection_with_directory(test_session, test assert collection.directory == domain_dir -@pytest.mark.asyncio -async def test_get_archive_destination_with_directory(test_session, test_directory): - """get_archive_destination returns collection directory when set.""" - from modules.archive.lib import get_archive_destination - from wrolpi.common import get_media_directory - - domain_dir = get_media_directory() / 'archives' / 'tagged' / 'example.com' - domain_dir.mkdir(parents=True, exist_ok=True) - - collection = Collection( - name='example.com', - kind='domain', - directory=domain_dir - ) - test_session.add(collection) - test_session.commit() - - destination = get_archive_destination(collection) - - assert destination == domain_dir - assert destination.is_dir() - - -@pytest.mark.asyncio -async def test_get_archive_destination_unrestricted(test_session): - """get_archive_destination returns default path for unrestricted collection.""" - from modules.archive.lib import get_archive_destination, get_archive_directory - - collection = Collection( - name='example.com', - kind='domain', - directory=None # Unrestricted - ) - test_session.add(collection) - test_session.commit() - - destination = get_archive_destination(collection) - expected = get_archive_directory() / 'example.com' - - assert destination == expected - - @pytest.mark.asyncio async def test_tag_domain_without_directory_saves_config(test_session, test_directory, tag_factory, await_switches): """Tagging domain collection without directory should still save config.""" diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 5ec0db2c..d91190fc 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -10,6 +10,9 @@ from PIL import Image from pytz import utc from modules.archive import lib +from modules.archive.lib import ArchiveDownloaderConfigValidator +from modules.archive.lib import format_archive_filename +from modules.archive.lib import get_archive_downloader_config from modules.archive.lib import get_or_create_domain_collection, get_new_archive_files, delete_archives, \ model_archive_result, get_domains from modules.archive.models import Archive @@ -373,7 +376,7 @@ def test_get_new_archive_files_length(test_directory, fake_now): @skip_circleci -def test_get_new_archive_files(fake_now): +def test_get_new_archive_files(test_directory, fake_now): """Archive files have a specific format, so they are sorted by datetime, and are near each other.""" fake_now(datetime(2001, 1, 1)) archive_files = get_new_archive_files('https://example.com/two', None) @@ -405,16 +408,50 @@ def test_get_new_archive_files_with_destination(test_directory, fake_now): # Files should be in the custom destination, not archive/example.com assert str(archive_files.singlefile).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.html') - assert str(archive_files.readability).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.html') - assert str(archive_files.readability_txt).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.txt') + assert str(archive_files.readability).endswith( + 'archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.html') + assert str(archive_files.readability_txt).endswith( + 'archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.txt') assert str(archive_files.readability_json).endswith( 'archive/News/custom-domain.com/2001-01-01-00-00-00_Title.readability.json') assert str(archive_files.screenshot).endswith('archive/News/custom-domain.com/2001-01-01-00-00-00_Title.png') +@skip_circleci +def test_get_new_archive_files_with_subdirectory_format(test_directory, fake_now): + """Archive files use subdirectories when the config format contains subdirectories.""" + from modules.archive.lib import get_archive_downloader_config + + fake_now(datetime(2001, 1, 1)) + + # Configure a file format with subdirectory + config = get_archive_downloader_config() + original_format = config._config['file_name_format'] + config._config['file_name_format'] = '%(download_year)s/%(download_datetime)s_%(title)s' + + try: + archive_files = get_new_archive_files('https://example.com/page', 'My Article') + + # Files should be created in a year subdirectory + assert str(archive_files.singlefile).endswith('archive/example.com/2001/2001-01-01-00-00-00_My Article.html') + assert str(archive_files.readability).endswith( + 'archive/example.com/2001/2001-01-01-00-00-00_My Article.readability.html') + assert str(archive_files.readability_txt).endswith( + 'archive/example.com/2001/2001-01-01-00-00-00_My Article.readability.txt') + assert str(archive_files.readability_json).endswith( + 'archive/example.com/2001/2001-01-01-00-00-00_My Article.readability.json') + assert str(archive_files.screenshot).endswith('archive/example.com/2001/2001-01-01-00-00-00_My Article.png') + + # The subdirectory should exist + subdir = test_directory / 'archive/example.com/2001' + assert subdir.is_dir() + finally: + config._config['file_name_format'] = original_format + + @skip_circleci @pytest.mark.asyncio -async def test_title_in_filename(test_session, fake_now, test_directory, image_bytes_factory): +async def test_title_in_filename(async_client, test_session, fake_now, test_directory, image_bytes_factory): """ The Archive files have the title in the path. """ @@ -457,6 +494,8 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b assert str(archive2.screenshot_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_NA.png' + # Test with malformed HTML - not properly closed with + # The parser includes in the title, which gets escaped singlefile = '\ntest single-file\nジにてこちら\ndangerous ;\\//_title</html></html>' readability = dict( # No Title from Readability. @@ -467,16 +506,18 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b archive3 = await model_archive_result('https://example.com', singlefile, readability, screenshot) + # Title includes </html></html> because the <title> tag is malformed (no ) + # Slashes are escaped: -> <⧸html> assert str(archive3.singlefile_path.relative_to(test_directory)) == \ - 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title.html' + 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title⧸html⧸html.html' assert str(archive3.readability_path.relative_to(test_directory)) == \ - 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title.readability.html' + 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title⧸html⧸html.readability.html' assert str(archive3.readability_txt_path.relative_to(test_directory)) == \ - 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title.readability.txt' + 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title⧸html⧸html.readability.txt' assert str(archive3.readability_json_path.relative_to(test_directory)) == \ - 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title.readability.json' + 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title⧸html⧸html.readability.json' assert str(archive3.screenshot_path.relative_to(test_directory)) == \ - 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title.png' + 'archive/example.com/2000-01-01-00-00-00_dangerous ;⧸⧸_title⧸html⧸html.png' @pytest.mark.asyncio @@ -880,7 +921,6 @@ async def test_get_custom_archive_directory(async_client, test_directory, test_w get_wrolpi_config().archive_destination = 'custom/archives' assert lib.get_archive_directory() == (test_directory / 'custom/archives') - assert lib.get_domain_directory('https://example.com') == (test_directory / 'custom/archives/example.com') @pytest.mark.asyncio @@ -1102,7 +1142,7 @@ def test_link_domain_and_downloads(test_session, test_download_manager): @skip_circleci @pytest.mark.asyncio async def test_archive_download_uses_domain_collection_directory(async_client, test_session, test_directory, fake_now, - monkeypatch): + monkeypatch): """ Test that archive downloads use the domain collection's directory when no explicit destination is set. @@ -1218,3 +1258,67 @@ async def test_archive_download_explicit_destination_overrides_domain_collection f"Archive should be in explicit destination {explicit_dir}, but was in {archive.singlefile_path.parent}" assert not str(archive.singlefile_path).startswith(str(collection_dir)), \ f"Archive should NOT be in collection directory {collection_dir}" + + +def test_archive_downloader_config(test_directory): + """ArchiveDownloaderConfig loads and provides file_name_format.""" + config = get_archive_downloader_config() + + # Default format should contain %(title)s + assert '%(title)s' in config.file_name_format + assert config.file_name_format == '%(download_datetime)s_%(title)s' + + +def test_archive_downloader_config_validation(): + """ArchiveDownloaderConfig validator rejects invalid formats.""" + # Valid format + ArchiveDownloaderConfigValidator(file_name_format='%(download_date)s_%(title)s') + + # Invalid format (missing %(title)s) + with pytest.raises(ValueError, match='must contain'): + ArchiveDownloaderConfigValidator(file_name_format='%(download_date)s_no_title') + + +def test_format_archive_filename_default(test_directory, fake_now): + """format_archive_filename uses default format.""" + result = format_archive_filename('My Article') + + # Default format: %(download_datetime)s_%(title)s + assert result == '2000-01-01-00-00-00_My Article' + + +def test_format_archive_filename_with_domain(test_directory, fake_now): + """format_archive_filename includes domain variable.""" + # Temporarily change the format to include domain + config = get_archive_downloader_config() + original_format = config._config['file_name_format'] + config._config['file_name_format'] = '%(domain)s_%(title)s' + + try: + result = format_archive_filename('My Article', domain='example.com') + assert result == 'example.com_My Article' + finally: + config._config['file_name_format'] = original_format + + +def test_format_archive_filename_with_year_subdirectory(test_directory, fake_now): + """format_archive_filename can include subdirectories in format.""" + config = get_archive_downloader_config() + original_format = config._config['file_name_format'] + config._config['file_name_format'] = '%(download_year)s/%(download_date)s_%(title)s' + + try: + result = format_archive_filename('My Article') + # Should include year subdirectory + assert result == '2000/2000-01-01_My Article' + finally: + config._config['file_name_format'] = original_format + + +def test_format_archive_filename_escapes_special_chars(test_directory, fake_now): + """format_archive_filename escapes special characters in title.""" + result = format_archive_filename('My/Article:With*Special?Chars') + + # Special characters should be escaped + assert '/' not in result.split('/')[-1] # No slashes in filename part + assert '2000-01-01' in result diff --git a/modules/videos/lib.py b/modules/videos/lib.py index f854d4b2..ae4cafe0 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -837,6 +837,10 @@ def format_videos_destination(channel_name: str = None, channel_tag: str = None, -> pathlib.Path: """Return the directory where Videos should be downloaded according to the WROLPi Config. + Note: When you have a Collection object, prefer using Collection.format_destination() + or Collection.get_or_set_directory() instead, as they provide unified behavior + across both channel and domain collections. + @warning: Directory may or may not exist.""" videos_destination = get_wrolpi_config().videos_destination diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index d5eed96e..b8eac1bc 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -824,24 +824,88 @@ class Collection(ModelHelper, Base): For domain collections (kind == 'domain'), use the configured archive directory: archive// + + @deprecated: Use format_destination() instead for new code. """ + # Delegate to format_destination for consistency + return self.format_destination(tag_name) + + def format_destination(self, tag_name: Optional[str] = None) -> pathlib.Path: + """Format the collection's destination path using config template. + + Uses videos_destination for kind='channel', archive_destination for kind='domain'. + Supports unified variables that work for both kinds. + + Args: + tag_name: Optional tag name to include in the path + + Returns: + Absolute path where this collection's files should be stored + """ + from wrolpi.common import get_wrolpi_config, escape_file_name + + config = get_wrolpi_config() + if self.kind == 'channel': - # Local import to avoid circular import: collections -> videos -> collections - from modules.videos.lib import format_videos_destination - return format_videos_destination(self.name, tag_name, None) + template = config.videos_destination elif self.kind == 'domain': - # Use configured archive directory for domain collections - # Local import to avoid circular import: collections -> archive -> collections - from modules.archive.lib import get_archive_directory - base = get_archive_directory() - if tag_name: - return base / tag_name / self.name - return base / self.name - # Default behavior: place under media root by kind/tag/name - base = get_media_directory() / self.kind - if tag_name: - return base / tag_name / self.name - return base / self.name + template = config.archive_destination + else: + template = '%(kind)s/%(tag)s/%(name)s' + + escaped_name = escape_file_name(self.name) if self.name else '' + + variables = dict( + # Unified variables + name=escaped_name, + tag=tag_name or '', + kind=self.kind or '', + # Backward compatibility aliases for videos + channel_name=escaped_name, + channel_tag=tag_name or '', + # Backward compatibility aliases for archives + domain=escaped_name, + domain_tag=tag_name or '', + ) + + destination = template % variables + return get_media_directory() / destination.lstrip('/') + + def get_or_set_directory(self, session: Session, tag_name: Optional[str] = None) -> pathlib.Path: + """Get collection's directory, or format and save it on first use. + + - If collection has directory: return it + - If no directory: format from template, save to collection, return it + + This ensures all files for a collection go to the same place, even if + the template config changes later. + + Args: + session: Database session for committing changes + tag_name: Optional tag name to use when formatting (defaults to collection's tag) + + Returns: + Absolute path to the collection's directory + """ + if self.directory: + return get_media_directory() / self.directory + + # First use - format and save + tag_name = tag_name or (self.tag.name if self.tag else None) + destination = self.format_destination(tag_name) + + self.directory = get_relative_to_media_directory(destination) + session.commit() + + # Trigger config save for persistence + if self.kind == 'domain': + from modules.archive.lib import save_domains_config + save_domains_config.activate_switch() + elif self.kind == 'channel': + from modules.videos.lib import save_channels_config + save_channels_config.activate_switch() + + return destination async def move_collection(self, directory: pathlib.Path, session: Session, send_events: bool = False): """Move all files under this Collection's directory to a new directory. diff --git a/wrolpi/collections/test/test_api.py b/wrolpi/collections/test/test_api.py index e49ecdab..052e46c2 100644 --- a/wrolpi/collections/test/test_api.py +++ b/wrolpi/collections/test/test_api.py @@ -141,9 +141,14 @@ class TestCollectionTagInfoAPI: @pytest.mark.asyncio async def test_get_tag_info_suggests_directory( - self, async_client, test_session, test_directory + self, async_client, test_session, test_directory, test_wrolpi_config ): """Test that tag_info endpoint suggests a directory for a domain collection.""" + # Configure template to include tag + from wrolpi.common import get_wrolpi_config + config = get_wrolpi_config() + config.update({'archive_destination': 'archive/%(domain_tag)s/%(domain)s'}) + # Create a domain collection collection = Collection( name='example.com', @@ -174,9 +179,14 @@ class TestCollectionTagInfoAPI: @pytest.mark.asyncio async def test_get_tag_info_detects_domain_conflict( - self, async_client, test_session, test_directory + self, async_client, test_session, test_directory, test_wrolpi_config ): """Test that tag_info detects conflicts with existing domain collections.""" + # Configure template to include tag + from wrolpi.common import get_wrolpi_config + config = get_wrolpi_config() + config.update({'archive_destination': 'archive/%(domain_tag)s/%(domain)s'}) + # Create two domain collections collection1 = Collection( name='example.com', @@ -211,9 +221,14 @@ class TestCollectionTagInfoAPI: @pytest.mark.asyncio async def test_get_tag_info_allows_channel_domain_same_directory( - self, async_client, test_session, test_directory, channel_factory + self, async_client, test_session, test_directory, channel_factory, test_wrolpi_config ): """Test that channel and domain collections can share directories.""" + # Configure template to include tag + from wrolpi.common import get_wrolpi_config + config = get_wrolpi_config() + config.update({'archive_destination': 'archive/%(domain_tag)s/%(domain)s'}) + # Create a channel collection channel = channel_factory(name='test', directory=test_directory / 'videos' / 'WROL' / 'test') test_session.flush() @@ -242,9 +257,14 @@ class TestCollectionTagInfoAPI: @pytest.mark.asyncio async def test_get_tag_info_detects_exact_directory_conflict( - self, async_client, test_session, test_directory + self, async_client, test_session, test_directory, test_wrolpi_config ): """Test that tag_info detects conflicts when suggested directory exactly matches existing collection.""" + # Configure template to include tag + from wrolpi.common import get_wrolpi_config + config = get_wrolpi_config() + config.update({'archive_destination': 'archive/%(domain_tag)s/%(domain)s'}) + # Create two domain collections with different names but where one's suggested directory # would conflict with the other's existing directory # collection1 is already in the "WROL" tagged location diff --git a/wrolpi/collections/test/test_models.py b/wrolpi/collections/test/test_models.py index bcccd42e..8a886542 100644 --- a/wrolpi/collections/test/test_models.py +++ b/wrolpi/collections/test/test_models.py @@ -2,6 +2,7 @@ import pytest from wrolpi.collections import Collection from wrolpi.collections.errors import UnknownCollection +from wrolpi.common import get_wrolpi_config def test_collection_get_by_id(test_session, test_directory): @@ -34,3 +35,109 @@ def test_collection_find_by_id(test_session, test_directory): # Non-existent ID should raise UnknownCollection with pytest.raises(UnknownCollection): Collection.find_by_id(test_session, 99999) + + +def test_collection_format_destination_channel(test_session, test_directory): + """Collection.format_destination formats path for channel collections.""" + collection = Collection(name='My Channel', kind='channel') + test_session.add(collection) + test_session.flush() + + destination = collection.format_destination() + # Should use videos_destination template: videos/%(channel_tag)s/%(channel_name)s + assert destination == test_directory / 'videos' / 'My Channel' + + # With tag + destination_with_tag = collection.format_destination(tag_name='News') + assert destination_with_tag == test_directory / 'videos' / 'News' / 'My Channel' + + +def test_collection_format_destination_domain(test_session, test_directory): + """Collection.format_destination formats path for domain collections.""" + collection = Collection(name='example.com', kind='domain') + test_session.add(collection) + test_session.flush() + + destination = collection.format_destination() + # Should use archive_destination template: archive/%(domain)s + assert destination == test_directory / 'archive' / 'example.com' + + # With tag - default template doesn't include %(domain_tag)s, so tag has no effect + destination_with_tag = collection.format_destination(tag_name='Tech') + # Default archive_destination is 'archive/%(domain)s' (no tag placeholder) + assert destination_with_tag == test_directory / 'archive' / 'example.com' + + +@pytest.mark.asyncio +async def test_collection_format_destination_domain_with_tag_template(async_client, test_session, test_directory, + test_wrolpi_config): + """Collection.format_destination uses tag when template includes %(domain_tag)s.""" + # Update config to include tag in template + config = get_wrolpi_config() + config.update({'archive_destination': 'archive/%(domain_tag)s/%(domain)s'}) + + collection = Collection(name='example.com', kind='domain') + test_session.add(collection) + test_session.flush() + + # With tag - template now includes %(domain_tag)s + destination_with_tag = collection.format_destination(tag_name='Tech') + assert destination_with_tag == test_directory / 'archive' / 'Tech' / 'example.com' + + # Without tag - empty tag component + destination_no_tag = collection.format_destination() + assert destination_no_tag == test_directory / 'archive' / 'example.com' + + +def test_collection_get_or_set_directory_existing(test_session, test_directory): + """Collection.get_or_set_directory returns existing directory without modifying it.""" + existing_dir = test_directory / 'existing/path' + collection = Collection(name='example.com', kind='domain', directory=existing_dir) + test_session.add(collection) + test_session.commit() + + result = collection.get_or_set_directory(test_session) + + assert result == existing_dir + # Directory should not have changed + assert collection.directory == existing_dir + + +@pytest.mark.asyncio +async def test_collection_get_or_set_directory_new(async_client, test_session, test_directory): + """Collection.get_or_set_directory formats and saves directory on first use.""" + collection = Collection(name='example.com', kind='domain') + test_session.add(collection) + test_session.commit() + + # Initially no directory + assert collection.directory is None + + result = collection.get_or_set_directory(test_session) + + # Should have set the directory + assert collection.directory is not None + assert result == test_directory / 'archive' / 'example.com' + # Directory should end with the relative path structure + assert str(collection.directory).endswith('archive/example.com') + + +@pytest.mark.asyncio +async def test_collection_get_or_set_directory_with_tag(async_client, test_session, test_directory, tag_factory, + test_wrolpi_config): + """Collection.get_or_set_directory uses collection's tag when formatting.""" + # Update config to include tag in template + config = get_wrolpi_config() + config.update({'archive_destination': 'archive/%(domain_tag)s/%(domain)s'}) + + tag = await tag_factory('News') + collection = Collection(name='example.com', kind='domain', tag=tag) + test_session.add(collection) + test_session.commit() + + result = collection.get_or_set_directory(test_session) + + # Should include tag in path (config has %(domain_tag)s) + assert result == test_directory / 'archive' / 'News' / 'example.com' + # Directory should end with the relative path structure + assert str(collection.directory).endswith('archive/News/example.com') diff --git a/wrolpi/common.py b/wrolpi/common.py index 15e7db57..94f10f43 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -25,7 +25,7 @@ from itertools import islice, filterfalse, tee from multiprocessing.managers import DictProxy from pathlib import Path from types import GeneratorType -from typing import Optional, List +from typing import Optional, List, Any, AsyncGenerator from typing import Union, Callable, Tuple, Dict, Iterable, Generator, Any, Set from urllib.parse import urlparse, urlunsplit @@ -764,9 +764,11 @@ def get_all_configs() -> Dict[str, ConfigFile]: if download_manager_config := get_download_manager_config(): all_configs[download_manager_config.file_name] = download_manager_config - from modules.archive.lib import get_domains_config + from modules.archive.lib import get_domains_config, get_archive_downloader_config if domains_config := get_domains_config(): all_configs[domains_config.file_name] = domains_config + if archive_downloader_config := get_archive_downloader_config(): + all_configs[archive_downloader_config.file_name] = archive_downloader_config return all_configs @@ -1460,7 +1462,7 @@ def recursive_map(obj: Any, func: callable): @contextlib.asynccontextmanager -async def aiohttp_session(timeout: int = None) -> ClientSession: +async def aiohttp_session(timeout: int = None) -> AsyncGenerator[ClientSession, Any]: """Convenience function because aiohttp timeout cannot be None.""" if timeout: timeout = aiohttp.ClientTimeout(total=timeout) if timeout else None @@ -1472,7 +1474,8 @@ async def aiohttp_session(timeout: int = None) -> ClientSession: @contextlib.asynccontextmanager -async def aiohttp_post(url: str, json_, timeout: int = None, headers: dict = None) -> ClientResponse: +async def aiohttp_post(url: str, json_, timeout: int = None, headers: dict = None) -> AsyncGenerator[ + ClientResponse, Any]: """Perform an async aiohttp POST request. Yield the response in a session.""" headers = headers or DEFAULT_HTTP_HEADERS async with aiohttp_session(timeout) as session: @@ -1481,7 +1484,7 @@ async def aiohttp_post(url: str, json_, timeout: int = None, headers: dict = Non @contextlib.asynccontextmanager -async def aiohttp_get(url: str, timeout: int = None, headers: dict = None) -> ClientResponse: +async def aiohttp_get(url: str, timeout: int = None, headers: dict = None) -> AsyncGenerator[ClientResponse, Any]: """Perform an async aiohttp GET request. Yield the response in a session.""" headers = headers or DEFAULT_HTTP_HEADERS async with aiohttp_session(timeout) as session: @@ -1490,7 +1493,7 @@ async def aiohttp_get(url: str, timeout: int = None, headers: dict = None) -> Cl @contextlib.asynccontextmanager -async def aiohttp_head(url: str, timeout: int = None, headers: dict = None) -> ClientResponse: +async def aiohttp_head(url: str, timeout: int = None, headers: dict = None) -> AsyncGenerator[ClientResponse, Any]: """Perform an async aiohttp HEAD request. Yield the response in a session.""" headers = headers or DEFAULT_HTTP_HEADERS async with aiohttp_session(timeout) as session: @@ -2266,6 +2269,11 @@ def create_empty_config_files() -> list[str]: get_videos_downloader_config().save(overwrite=True) created.append(get_videos_downloader_config().get_file().name) + from modules.archive.lib import get_archive_downloader_config + if not get_archive_downloader_config().get_file().is_file(): + get_archive_downloader_config().save(overwrite=True) + created.append(get_archive_downloader_config().get_file().name) + if not get_inventories_config().get_file().is_file(): from modules.inventory.models import Inventory if session.query(Inventory).count() == 0: diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index ab6f43c1..1c9e505b 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -30,6 +30,7 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.download_manager_config = manager.dict() app.shared_ctx.videos_downloader_config = manager.dict() app.shared_ctx.domains_config = manager.dict() + app.shared_ctx.archive_downloader_config = manager.dict() # Shared dicts. app.shared_ctx.refresh = manager.dict() app.shared_ctx.uploaded_files = manager.dict() @@ -90,6 +91,7 @@ def reset_shared_contexts(app: Sanic): app.shared_ctx.download_manager_config.clear() app.shared_ctx.videos_downloader_config.clear() app.shared_ctx.domains_config.clear() + app.shared_ctx.archive_downloader_config.clear() # Shared dicts. app.shared_ctx.refresh.clear() app.shared_ctx.uploaded_files.clear() @@ -209,3 +211,9 @@ def initialize_configs_contexts(app: Sanic): domains_config.initialize(app.shared_ctx.domains_config) except Exception as e: logger.error(f'Failed to initialize in-memory domains config: {e}') + + try: + from modules.archive.lib import ARCHIVE_DOWNLOADER_CONFIG + ARCHIVE_DOWNLOADER_CONFIG.initialize(app.shared_ctx.archive_downloader_config) + except Exception as e: + logger.error(f'Failed to initialize in-memory archive downloader config: {e}') diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index ad3d6b1a..708d78db 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -900,7 +900,8 @@ count = 0 @pytest.mark.asyncio -async def test_move_error(test_session, test_directory, make_files_structure, video_bytes, singlefile_contents_factory): +async def test_move_error(async_client, test_session, test_directory, make_files_structure, video_bytes, + singlefile_contents_factory): """Files are restored when a move fails.""" make_files_structure({ 'foo/bar/video.mp4': video_bytes, diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 9cfc8217..e8dbdbb7 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1067,6 +1067,7 @@ async def test_initialize_config_files(async_client, test_session, test_director test_download_manager, test_downloader): """Config files should only be created if they are missing, and they will not overwrite existing entries in the DB.""" from modules.videos.lib import get_videos_downloader_config, get_channels_config + from modules.archive.lib import get_archive_downloader_config from wrolpi.tags import get_tags_config from wrolpi.downloader import get_download_manager_config from modules.inventory.common import get_inventories_config @@ -1079,11 +1080,13 @@ async def test_initialize_config_files(async_client, test_session, test_director # Write some config files, they should not be re-created. get_wrolpi_config().get_file().write_text('wrolpi') get_videos_downloader_config().get_file().write_text('video downloader') + get_archive_downloader_config().get_file().write_text('archive downloader') assert common.create_empty_config_files() == [], 'No configs should have been created.' # Config files were not overwritten with real configs. assert get_wrolpi_config().get_file().read_text() == 'wrolpi' assert get_videos_downloader_config().get_file().read_text() == 'video downloader' + assert get_archive_downloader_config().get_file().read_text() == 'archive downloader' # WROLPi config can be created because it is now missing. get_wrolpi_config().get_file().unlink() @@ -1096,6 +1099,12 @@ async def test_initialize_config_files(async_client, test_session, test_director 'Videos Downloader config should have been created.' assert get_videos_downloader_config().is_valid() + # Archive Downloader config can be created because it is now missing. + get_archive_downloader_config().get_file().unlink() + assert common.create_empty_config_files() == ['archives_downloader.yaml'], \ + 'Archive Downloader config should have been created.' + assert get_archive_downloader_config().is_valid() + # Channels config file does not exist, no conflicting Channels so the file is created. simple_channel.delete_with_videos() assert common.create_empty_config_files() == ['channels.yaml'], 'Channels config should have been created.'