diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index 0fb92110..5637096e 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -1,11 +1,26 @@ import React from "react"; import {getSettings, postRestart, postShutdown, saveSettings} from "../../api"; -import {Button, Form, FormGroup, FormInput, Header, Loader, Modal, ModalContent, ModalHeader, Segment} from "../Theme"; -import {Container, Dimmer, Icon} from "semantic-ui-react"; +import { + Button, + Divider, + Form, + FormField, + FormGroup, + FormInput, + Header, + Loader, + Modal, + ModalContent, + ModalHeader, + Segment +} from "../Theme"; +import {Container, Dimmer, GridColumn, GridRow, Icon, Input} from "semantic-ui-react"; import {APIButton, ErrorMessage, HelpPopup, HotspotToggle, ThrottleToggle, Toggle, WROLModeMessage} from "../Common"; import QRCode from "react-qr-code"; import {useDockerized} from "../../hooks/customHooks"; import {toast} from "react-semantic-toasts-2"; +import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; +import {SettingsContext} from "../../contexts/contexts"; export function ShutdownButton() { const dockerized = useDockerized(); @@ -151,11 +166,13 @@ export class SettingsPage extends React.Component { this.state = { disabled: false, hotspot_encryption: 'WPA', + edit_special_directories: false, pending: false, qrCodeValue: '', qrOpen: false, ready: false, + archive_directory: null, download_on_startup: null, download_timeout: null, hotspot_device: null, @@ -164,8 +181,11 @@ export class SettingsPage extends React.Component { hotspot_status: null, ignore_outdated_zims: null, log_level: null, + map_directory: null, throttle_on_startup: null, throttle_status: null, + videos_directory: null, + zims_directory: null, } this.handleSubmit = this.handleSubmit.bind(this); @@ -177,6 +197,7 @@ export class SettingsPage extends React.Component { const settings = await getSettings(); this.setState({ ready: true, + archive_directory: settings.archive_directory, disabled: settings.wrol_mode, download_on_startup: settings.download_on_startup, download_timeout: settings.download_timeout || '', @@ -186,8 +207,11 @@ export class SettingsPage extends React.Component { hotspot_status: settings.hotspot_status, ignore_outdated_zims: settings.ignore_outdated_zims, log_level: fromApiLogLevel(settings.log_level), + map_directory: settings.map_directory, throttle_on_startup: settings.throttle_on_startup, throttle_status: settings.throttle_status, + videos_directory: settings.videos_directory, + zims_directory: settings.zims_directory, }, this.handleHotspotChange); } catch (e) { console.error(e); @@ -201,6 +225,7 @@ export class SettingsPage extends React.Component { } this.setState({disabled: true, pending: true}); let settings = { + archive_directory: this.state.archive_directory, download_on_startup: this.state.download_on_startup, download_timeout: this.state.download_timeout ? parseInt(this.state.download_timeout) : 0, hotspot_device: this.state.hotspot_device, @@ -208,7 +233,10 @@ export class SettingsPage extends React.Component { hotspot_ssid: this.state.hotspot_ssid, ignore_outdated_zims: this.state.ignore_outdated_zims, log_level: toApiLogLevel(this.state.log_level), + map_directory: this.state.map_directory, throttle_on_startup: this.state.throttle_on_startup, + videos_directory: this.state.videos_directory, + zims_directory: this.state.zims_directory, } try { const response = await saveSettings(settings); @@ -263,157 +291,231 @@ export class SettingsPage extends React.Component { return Unable to fetch settings } - let { + const { + archive_directory, disabled, download_on_startup, download_timeout, + edit_special_directories, hotspot_device, hotspot_password, hotspot_ssid, ignore_outdated_zims, log_level, + map_directory, pending, qrCodeValue, throttle_on_startup, + videos_directory, + zims_directory, } = this.state; const qrButton = ; - return - + return {({settings}) => { + const mediaDirectoryLabel = `${settings.media_directory}/`; + return + - - - - + + + + - -
- Settings -
-

Any changes will be written to config/wrolpi.yaml.

+ +
Settings
+

Any changes will be written to {settings.media_directory}/config/wrolpi.yaml.

-
-
- this.handleInputChange(null, 'download_on_startup', checked)} - /> -
+ +
+ this.handleInputChange(null, 'download_on_startup', checked)} + /> +
-
- this.handleInputChange(null, 'throttle_on_startup', checked)} - /> -
+
+ this.handleInputChange(null, 'throttle_on_startup', checked)} + /> +
-
- this.handleInputChange(null, 'ignore_outdated_zims', checked)} - /> -
+
+ this.handleInputChange(null, 'ignore_outdated_zims', checked)} + /> +
- - - Download Timeout - - } - value={download_timeout} - disabled={disabled || download_timeout === null} - onChange={(e, d) => this.handleTimeoutChange(e, 'download_timeout', d.value)} + } + value={download_timeout} + disabled={disabled || download_timeout === null} + onChange={(e, d) => this.handleTimeoutChange(e, 'download_timeout', d.value)} + /> + + + + this.setState({hotspot_ssid: d.value}, this.handleHotspotChange)} + /> + this.setState({hotspot_password: d.value}, this.handleHotspotChange)} + /> + this.handleInputChange(e, 'hotspot_device', d.value)} + /> + + + this.setState({qrOpen: false})} + onOpen={this.handleQrOpen} + open={this.state.qrOpen} + trigger={qrButton} + > + + Scan this code to join the hotspot + + +
+ +
+ + + +
+ + +
+ this.setState({'log_level': parseInt(e.target.value)})} + style={{marginBottom: '1em'}} /> - + + + + + + + - - this.setState({hotspot_ssid: d.value}, this.handleHotspotChange)} - /> - this.setState({hotspot_password: d.value}, this.handleHotspotChange)} - /> - this.handleInputChange(e, 'hotspot_device', d.value)} - /> - + - this.setState({qrOpen: false})} - onOpen={this.handleQrOpen} - open={this.state.qrOpen} - trigger={qrButton} - > - - Scan this code to join the hotspot - - -
- -
- - +
Special Directories
+

WROLPi will save files to these directories (any video files will be saved to + the videos by default, etc.).

-
+ + + + + + this.handleInputChange(e, 'archive_directory', d.value)} + /> + + + + + + this.handleInputChange(e, 'videos_directory', d.value)} + /> + + + + + + + + this.handleInputChange(e, 'map_directory', d.value)} + /> + + + + + + this.handleInputChange(e, 'zims_directory', d.value)} + /> + + + + + + this.handleInputChange(null, 'edit_special_directories', checked)} + /> + + + - -
- this.setState({'log_level': parseInt(e.target.value)})} - style={{marginBottom: '1em'}} - /> - - - - - - - + -
+ Save - Save + + + - - - + +
- - - - -
Control WROLPi
- - - - + +
Control WROLPi
+ + + + ; + } + } } } diff --git a/main.py b/main.py index aa0cbc01..6330edcb 100755 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ from sanic.signals import Event from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS from wrolpi import root_api, admin -from wrolpi.common import logger, get_config, check_media_directory, limit_concurrent, \ +from wrolpi.common import logger, get_wrolpi_config, check_media_directory, limit_concurrent, \ wrol_mode_enabled, cancel_refresh_tasks, set_log_level, background_task, cancel_background_tasks from wrolpi.dates import Seconds from wrolpi.downloader import download_manager, import_downloads_config @@ -136,7 +136,7 @@ def main(): if args.sub_commands == 'db': return db_main(args) - config = get_config() + config = get_wrolpi_config() # Hotspot/throttle are not supported in Docker containers. if not DOCKERIZED and config.hotspot_on_startup: @@ -246,7 +246,7 @@ async def periodic_downloads(app: Sanic): download_manager.disable() return - config = get_config() + config = get_wrolpi_config() if config.download_on_startup is False: logger.warning('Not starting download manager because Downloads are disabled on startup.') download_manager.disable() diff --git a/modules/archive/lib.py b/modules/archive/lib.py index d1bd993d..31e697d2 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -15,7 +15,7 @@ from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive from wrolpi import dates from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \ - format_html_string, split_lines_by_length, get_html_soup, get_title_from_html + format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config from wrolpi.dates import now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime @@ -48,7 +48,7 @@ class ArchiveFiles: def get_archive_directory() -> pathlib.Path: - archive_directory = get_media_directory() / 'archive' + archive_directory = get_media_directory() / get_wrolpi_config().archive_directory return archive_directory diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index e22a80ca..330aefbd 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -13,6 +13,7 @@ from modules.archive import lib from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, model_archive_result, \ get_domains from modules.archive.models import Archive, Domain +from wrolpi.common import get_wrolpi_config from wrolpi.db import get_db_session from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup @@ -817,3 +818,14 @@ SINGLEFILE_EXAMPLE_1 = b''' def test_get_url_from_singlefile(): assert lib.get_url_from_singlefile(SINGLEFILE_EXAMPLE_1) == 'https://www.example.com' + + +def test_get_custom_archive_directory(test_directory, test_config): + """Custom directory can be used for archive directory.""" + # Default location. + assert lib.get_archive_directory() == (test_directory / 'archive') + + get_wrolpi_config().archive_directory = 'custom/archives' + + assert lib.get_archive_directory() == (test_directory / 'custom/archives') + assert lib.get_domain_directory('https://example.com') == (test_directory / 'custom/archives/example.com') diff --git a/modules/map/lib.py b/modules/map/lib.py index a9541fae..dd025794 100644 --- a/modules/map/lib.py +++ b/modules/map/lib.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import Session from modules.map.models import MapFile from wrolpi import flags from wrolpi.cmd import SUDO_BIN -from wrolpi.common import get_media_directory, walk, logger +from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp from wrolpi.db import optional_session, get_db_session from wrolpi.events import Events @@ -23,14 +23,14 @@ IMPORTING = Manager().dict(dict( def get_map_directory() -> Path: - map_directory = get_media_directory() / 'map' + map_directory = get_media_directory() / get_wrolpi_config().map_directory if not map_directory.is_dir(): - map_directory.mkdir() + map_directory.mkdir(parents=True) return map_directory def is_pbf_file(pbf: Path) -> bool: - """Uses file command to check type of a file. Returns True if a file is an OpenStreetMap PBF file.""" + """Uses file command to check type of file. Returns True if a file is an OpenStreetMap PBF file.""" cmd = ('/usr/bin/file', pbf) try: output = subprocess.check_output(cmd) diff --git a/modules/map/test/test_lib.py b/modules/map/test/test_lib.py index 7069b404..e5233a2f 100644 --- a/modules/map/test/test_lib.py +++ b/modules/map/test/test_lib.py @@ -4,6 +4,7 @@ from unittest import mock import pytest from modules.map import lib +from wrolpi.common import get_wrolpi_config def test_get_map_paths(test_directory, make_files_structure): @@ -101,3 +102,14 @@ def test_seconds_to_import_rpi4(size, expected): def test_seconds_to_import_rpi5(size, expected): with unittest.mock.patch('modules.map.lib.IS_RPI5', True): assert lib.seconds_to_import(size) == expected + + +def test_get_custom_map_directory(test_directory, test_config): + """Custom directory can be used for map directory.""" + # Default location. + assert lib.get_map_directory() == (test_directory / 'map') + + get_wrolpi_config().map_directory = 'custom/deep/map/directory' + + assert lib.get_map_directory() == (test_directory / 'custom/deep/map/directory') + assert (test_directory / 'custom/deep/map/directory').is_dir() diff --git a/modules/videos/common.py b/modules/videos/common.py index 207d919d..c25bf211 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -17,7 +17,7 @@ from sqlalchemy.orm import Session from wrolpi.captions import FFMPEG_BIN from wrolpi.cmd import FFPROBE_BIN -from wrolpi.common import logger, get_media_directory +from wrolpi.common import logger, get_media_directory, get_wrolpi_config from wrolpi.dates import seconds_to_timestamp from wrolpi.db import get_db_session, get_db_curs from wrolpi.vars import DEFAULT_FILE_PERMISSIONS @@ -35,7 +35,8 @@ class ConfigError(Exception): def get_videos_directory() -> pathlib.Path: """Get the "videos" directory in the media directory. Make it if it does not exist.""" - directory = get_media_directory() / 'videos' + config = get_wrolpi_config() + directory = get_media_directory() / config.videos_directory if not directory.is_dir(): directory.mkdir(parents=True) return directory diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 0912643a..9fe817c2 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -7,7 +7,7 @@ import pytest from PIL import Image from modules.videos.models import Channel, Video -from wrolpi.common import get_absolute_media_path +from wrolpi.common import get_absolute_media_path, get_wrolpi_config from wrolpi.downloader import Download, DownloadFrequency from wrolpi.files import lib as files_lib from wrolpi.vars import PROJECT_DIR @@ -273,3 +273,23 @@ async def test_video_ffprobe_json(test_session, video_file): video = test_session.query(Video).one() assert video.ffprobe_json assert video.file_group.length + + +def test_get_videos_directory(test_directory): + """Default videos directory path""" + # Directory does not yet exist. + assert not (test_directory / 'videos').exists() + + # Directory is created when first gotten. + assert common.get_videos_directory() == (test_directory / 'videos') + assert (test_directory / 'videos').is_dir() + + +def test_get_custom_videos_directory(test_directory, test_config): + """Custom directory can be used for videos directory.""" + config = get_wrolpi_config() + config.videos_directory = 'custom/directory/videos' + + assert common.get_videos_directory() == (test_directory / 'custom/directory/videos') + assert (test_directory / 'custom/directory/videos').is_dir() + assert not (test_directory / 'videos').is_dir() diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 55a102a5..0b199afc 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -16,7 +16,7 @@ from modules.zim.kiwix import KIWIX_CATALOG from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription from wrolpi import flags from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \ - register_refresh_cleanup, background_task + register_refresh_cleanup, background_task, get_wrolpi_config from wrolpi.db import get_db_session, optional_session, get_db_curs from wrolpi.downloader import DownloadFrequency from wrolpi.files.lib import refresh_files, split_file_name_words @@ -464,8 +464,7 @@ def zim_download_url_to_name(url: str) -> Tuple[str, str]: def get_zim_directory() -> pathlib.Path: - media_directory = get_media_directory() - zim_directory = media_directory / 'zims' + zim_directory = get_media_directory() / get_wrolpi_config().zims_directory return zim_directory diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 731fb66b..2ac83f90 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -8,7 +8,7 @@ from modules.zim import lib from modules.zim.errors import UnknownZim from modules.zim.models import TagZimEntry, ZimSubscription from wrolpi import tags, flags -from wrolpi.common import DownloadFileInfo +from wrolpi.common import DownloadFileInfo, get_wrolpi_config from wrolpi.downloader import Download, import_downloads_config, save_downloads_config from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup @@ -341,3 +341,12 @@ async def test_zim_modeler(test_session, zim_factory): fg: FileGroup = test_session.query(FileGroup).one() assert fg.title == 'the_zim_title' assert fg.a_text == 'the zim title' + + +def test_get_custom_zims_directory(test_directory, test_config): + """Custom directory can be used for zims directory.""" + assert lib.get_zim_directory() == (test_directory / 'zims') + + get_wrolpi_config().zims_directory = 'custom/zims' + + assert lib.get_zim_directory() == (test_directory / 'custom/zims') diff --git a/wrolpi/common.py b/wrolpi/common.py index 54ec84bd..588f8c50 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -126,7 +126,7 @@ __all__ = [ 'extract_headlines', 'extract_html_text', 'get_absolute_media_path', - 'get_config', + 'get_wrolpi_config', 'get_download_info', 'get_files_and_directories', 'get_html_soup', @@ -212,6 +212,51 @@ def compile_tsvector(element, compiler, **kw): URL_CHARS = string.ascii_lowercase + string.digits +def find_file(directory: pathlib.Path, name: str, depth=1) -> Optional[pathlib.Path]: + """Recursively searches a directory for a file with the provided name.""" + if depth == 0: + return + + for path in sorted(directory.iterdir()): + if path.is_file() and path.name == name: + return path + if path.is_dir() and (result := find_file(path, name, depth - 1)): + return result + + +TEST_MEDIA_DIRECTORY = None + + +def set_test_media_directory(path): + global TEST_MEDIA_DIRECTORY + if isinstance(path, pathlib.Path): + TEST_MEDIA_DIRECTORY = path + elif path: + TEST_MEDIA_DIRECTORY = pathlib.Path(path) + else: + TEST_MEDIA_DIRECTORY = None + + +def get_media_directory() -> Path: + """Get the media directory. + + This will typically be /opt/wrolpi, or something else from the .env. + + During testing, this function returns TEST_MEDIA_DIRECTORY. + """ + global TEST_MEDIA_DIRECTORY + + if PYTEST and not TEST_MEDIA_DIRECTORY: + raise ValueError('No test media directory set during testing!!') + + if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path): + if not str(TEST_MEDIA_DIRECTORY).startswith('/tmp'): + raise RuntimeError('Refusing to run test outside tmp directory!') + return TEST_MEDIA_DIRECTORY + + return MEDIA_DIRECTORY + + class ConfigFile: """ This class keeps track of the contents of a config file. You can update the config (and it's file) by @@ -306,6 +351,7 @@ class ConfigFile: class WROLPiConfig(ConfigFile): file_name = 'wrolpi.yaml' default_config = dict( + archive_directory='archive', download_on_startup=True, download_timeout=0, hotspot_device='wlan0', @@ -314,10 +360,34 @@ class WROLPiConfig(ConfigFile): hotspot_ssid='WROLPi', ignore_outdated_zims=False, ignored_directories=list(), + map_directory='map', throttle_on_startup=False, + videos_directory='videos', wrol_mode=False, + zims_directory='zims', ) + def get_file(self) -> Path: + """WROLPiConfig must be discovered so that other config files can be found.""" + if not self.file_name: + raise NotImplementedError(f'You must define a file name for this {self.__class__.__name__} config.') + + if PYTEST: + return get_media_directory() / f'config/{self.file_name}' + + # Use the usual "/media/wrolpi/config/wrolpi.yaml" if it exists. + default_config_path = CONFIG_DIR / self.file_name + if default_config_path.is_file(): + return default_config_path + + # Search the media directory for the special "wrolpi.yaml" file. Assume the first one found is the config. + # Use a depth of 3; multiple drives may exist and the config directory may be down in a second drive. + if config_path := find_file(get_media_directory(), self.file_name, 3): + return config_path + + # Not testing, and can't find file deep in media directory. Use the default. + return default_config_path + @property def download_on_startup(self) -> bool: return self._config['download_on_startup'] @@ -398,12 +468,44 @@ class WROLPiConfig(ConfigFile): def ignored_directories(self, value: List[str]): self.update({'ignored_directories': value}) + @property + def videos_directory(self) -> str: + return self._config['videos_directory'] + + @videos_directory.setter + def videos_directory(self, value: str): + self.update({'videos_directory': value}) + + @property + def archive_directory(self) -> str: + return self._config['archive_directory'] + + @archive_directory.setter + def archive_directory(self, value: str): + self.update({'archive_directory': value}) + + @property + def map_directory(self) -> str: + return self._config['map_directory'] + + @map_directory.setter + def map_directory(self, value: str): + self.update({'map_directory': value}) + + @property + def zims_directory(self) -> str: + return self._config['zims_directory'] + + @zims_directory.setter + def zims_directory(self, value: str): + self.update({'zims_directory': value}) + WROLPI_CONFIG: WROLPiConfig = WROLPiConfig() TEST_WROLPI_CONFIG: WROLPiConfig = None -def get_config() -> WROLPiConfig: +def get_wrolpi_config() -> WROLPiConfig: """Read the global WROLPi yaml config file.""" global TEST_WROLPI_CONFIG if isinstance(TEST_WROLPI_CONFIG, WROLPiConfig): @@ -420,7 +522,7 @@ def set_test_config(enable: bool): def wrol_mode_enabled() -> bool: """Return True if WROL Mode is enabled.""" - return get_config().wrol_mode + return get_wrolpi_config().wrol_mode def wrol_mode_check(func): @@ -445,7 +547,7 @@ def enable_wrol_mode(download_manager=None): Stop downloads and Download Manager. """ logger_.warning('ENABLING WROL MODE') - get_config().wrol_mode = True + get_wrolpi_config().wrol_mode = True if not download_manager: from wrolpi.downloader import download_manager download_manager.stop() @@ -461,7 +563,7 @@ def disable_wrol_mode(download_manager=None): Start downloads and Download Manager. """ logger_.warning('DISABLING WROL MODE') - get_config().wrol_mode = False + get_wrolpi_config().wrol_mode = False if not download_manager: from wrolpi.downloader import download_manager download_manager.enable() @@ -549,39 +651,6 @@ def run_after(after: callable, *args, **kwargs) -> callable: return wrapper -TEST_MEDIA_DIRECTORY = None - - -def set_test_media_directory(path): - global TEST_MEDIA_DIRECTORY - if isinstance(path, pathlib.Path): - TEST_MEDIA_DIRECTORY = path - elif path: - TEST_MEDIA_DIRECTORY = pathlib.Path(path) - else: - TEST_MEDIA_DIRECTORY = None - - -def get_media_directory() -> Path: - """Get the media directory. - - This will typically be /opt/wrolpi, or something else from the .env. - - During testing, this function returns TEST_MEDIA_DIRECTORY. - """ - global TEST_MEDIA_DIRECTORY - - if PYTEST and not TEST_MEDIA_DIRECTORY: - raise ValueError('No test media directory set during testing!!') - - if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path): - if not str(TEST_MEDIA_DIRECTORY).startswith('/tmp'): - raise RuntimeError('Refusing to run test outside tmp directory!') - return TEST_MEDIA_DIRECTORY - - return MEDIA_DIRECTORY - - MEDIA_DIRECTORY_PERMISSIONS = 0o40755 @@ -1190,7 +1259,8 @@ def timer(name, level: str = 'debug'): try: yield finally: - log_method(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds') + elapsed = (datetime.now() - before).total_seconds() + log_method(f'{name} elapsed {elapsed} seconds') def async_timer(coro: Coroutine, name: str = 'async timer', level: str = 'debug') -> callable: diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 89250a0f..40d8aaac 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -23,7 +23,7 @@ from wrolpi import flags from wrolpi.cmd import which from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \ partition, cancelable_wrapper, \ - get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, get_config, \ + get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, get_wrolpi_config, \ timer from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session @@ -435,7 +435,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]: - ignored_directories = list(map(str, get_config().ignored_directories)) + ignored_directories = list(map(str, get_wrolpi_config().ignored_directories)) files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)] return files @@ -1323,7 +1323,7 @@ def add_ignore_directory(directory: Union[pathlib.Path, str]): if not str(directory).startswith(str(media_directory)): raise InvalidDirectory('Cannot ignore directory not in media directory') - wrolpi_config = get_config() + wrolpi_config = get_wrolpi_config() ignored_directories = wrolpi_config.ignored_directories if wrolpi_config.ignored_directories else list() if str(directory) in ignored_directories: @@ -1337,9 +1337,9 @@ def add_ignore_directory(directory: Union[pathlib.Path, str]): def remove_ignored_directory(directory: Union[pathlib.Path, str]): """Remove a directory from the `ignored_directories` in the WROLPi config.""" directory = str(pathlib.Path(directory)) - ignored_directories = get_config().ignored_directories + ignored_directories = get_wrolpi_config().ignored_directories if directory in ignored_directories: ignored_directories.remove(directory) - get_config().ignored_directories = ignored_directories + get_wrolpi_config().ignored_directories = ignored_directories else: raise UnknownDirectory('Directory is not ignored') diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 6438cf39..0dfd3d79 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -4,7 +4,7 @@ from unittest import mock import pytest -from wrolpi.common import get_config +from wrolpi.common import get_wrolpi_config from wrolpi.files import lib from wrolpi.files.models import FileGroup from wrolpi.tags import TagFile @@ -634,13 +634,13 @@ async def test_get_file(test_session, test_async_client, test_directory, make_fi async def test_ignore_directory(test_session, test_async_client, test_directory, make_files_structure, test_config): """A maintainer can ignore/un-ignore directories. The files in the directory should not be refreshed.""" foo, bar, baz = make_files_structure(['foo/foo.txt', 'foo/bar.txt', 'baz/baz.txt']) - assert len(get_config().ignored_directories) == 0 + assert len(get_wrolpi_config().ignored_directories) == 0 # Ignore baz/ content = dict(path=str(baz.parent)) request, response = await test_async_client.post('/api/files/ignore_directory', json=content) assert response.status_code == HTTPStatus.OK - assert len(get_config().ignored_directories) == 1 + assert len(get_wrolpi_config().ignored_directories) == 1 await lib.refresh_files() @@ -650,7 +650,7 @@ async def test_ignore_directory(test_session, test_async_client, test_directory, # Un-ignore baz/ request, response = await test_async_client.post('/api/files/unignore_directory', json=content) assert response.status_code == HTTPStatus.OK - assert len(get_config().ignored_directories) == 0 + assert len(get_wrolpi_config().ignored_directories) == 0 await lib.refresh_files() @@ -661,4 +661,4 @@ async def test_ignore_directory(test_session, test_async_client, test_directory, content = dict(path=str(test_directory / 'videos')) request, response = await test_async_client.post('/api/files/ignore_directory', json=content) assert response.status_code == HTTPStatus.BAD_REQUEST - assert len(get_config().ignored_directories) == 0 + assert len(get_wrolpi_config().ignored_directories) == 0 diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 9111fa21..2d38fe1b 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -1,5 +1,6 @@ import asyncio import json +import pathlib import re from datetime import datetime, date, timezone from decimal import Decimal @@ -21,7 +22,7 @@ from vininfo.details._base import VinDetails from wrolpi import admin, status, flags, schema, dates from wrolpi import tags from wrolpi.admin import HotspotStatus -from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \ +from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, Base, get_media_directory, \ wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, LOG_LEVEL, \ set_global_log_level, get_relative_to_media_directory from wrolpi.dates import now @@ -139,11 +140,12 @@ async def echo(request: Request): @openapi.description('Get WROLPi settings') @openapi.response(HTTPStatus.OK, schema.SettingsResponse) def get_settings(_: Request): - config = get_config() + config = get_wrolpi_config() ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories] settings = { + 'archive_directory': config.archive_directory, 'download_manager_disabled': download_manager.disabled.is_set(), 'download_manager_stopped': download_manager.stopped.is_set(), 'download_on_startup': config.download_on_startup, @@ -153,14 +155,17 @@ def get_settings(_: Request): 'hotspot_password': config.hotspot_password, 'hotspot_ssid': config.hotspot_ssid, 'hotspot_status': admin.hotspot_status().name, - 'log_level': LOG_LEVEL.value, 'ignore_outdated_zims': config.ignore_outdated_zims, 'ignored_directories': ignored_directories, + 'log_level': LOG_LEVEL.value, + 'map_directory': config.map_directory, 'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative. 'throttle_on_startup': config.throttle_on_startup, 'throttle_status': admin.throttle_status().name, 'version': __version__, + 'videos_directory': config.videos_directory, 'wrol_mode': config.wrol_mode, + 'zims_directory': config.zims_directory, } return json_response(settings) @@ -168,7 +173,7 @@ def get_settings(_: Request): @api_bp.patch('/settings') @openapi.description('Update WROLPi settings') @validate(json=schema.SettingsRequest) -def update_settings(request: Request, body: schema.SettingsRequest): +def update_settings(_: Request, body: schema.SettingsRequest): if wrol_mode_enabled() and body.wrol_mode is None: # Cannot update settings while WROL Mode is enabled, unless you want to disable WROL Mode. raise WROLModeEnabled() @@ -187,11 +192,31 @@ def update_settings(request: Request, body: schema.SettingsRequest): # Remove any keys with None values, then save the config. config = {k: v for k, v in body.__dict__.items() if v is not None} - wrolpi_config = get_config() + wrolpi_config = get_wrolpi_config() if not config: raise InvalidConfig() + if body.archive_directory and pathlib.Path(body.archive_directory).is_absolute(): + raise InvalidConfig('Archive directory must be relative to media directory') + elif not body.archive_directory: + config['archive_directory'] = wrolpi_config.default_config['archive_directory'] + + if body.videos_directory and pathlib.Path(body.videos_directory).is_absolute(): + raise InvalidConfig('Videos directory must be relative to media directory') + elif not body.videos_directory: + config['videos_directory'] = wrolpi_config.default_config['videos_directory'] + + if body.map_directory and pathlib.Path(body.map_directory).is_absolute(): + raise InvalidConfig('Map directory must be relative to media directory') + elif not body.map_directory: + config['map_directory'] = wrolpi_config.default_config['map_directory'] + + if body.zims_directory and pathlib.Path(body.zims_directory).is_absolute(): + raise InvalidConfig('Zims directory must be relative to media directory') + elif not body.zims_directory: + config['zims_directory'] = wrolpi_config.default_config['zims_directory'] + log_level = config.pop('log_level', None) if isinstance(log_level, int): set_global_log_level(log_level) diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 8d1b93bd..b0480f20 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -16,6 +16,7 @@ class SettingsResponse: @dataclass class SettingsRequest: + archive_directory: Optional[str] = None download_on_startup: Optional[bool] = None download_timeout: Optional[int] = None hotspot_device: Optional[str] = None @@ -25,10 +26,13 @@ class SettingsRequest: hotspot_status: Optional[bool] = None ignore_outdated_zims: Optional[bool] = None log_level: Optional[int] = None + map_directory: Optional[str] = None media_directory: Optional[str] = None throttle_on: Optional[bool] = None throttle_on_startup: Optional[bool] = None + videos_directory: Optional[str] = None wrol_mode: Optional[bool] = None + zims_directory: Optional[str] = None @dataclass diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 09f9f7bb..5e7a25da 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -778,3 +778,22 @@ def test_format_html_file(): ) def test_chain(iterable, length, step, expected): assert list(common.chain(iterable, length)) == expected + + +def test_find_file(test_directory, make_files_structure): + bar, baz, foo = make_files_structure({ + 'foo/foo.txt': 'foo', + 'bar/bar.txt': 'bar', + 'baz.txt': 'baz', + }) + + # Files are too deep. + assert common.find_file(test_directory, 'foo.txt') is None + assert common.find_file(test_directory, 'bar.txt') is None + assert common.find_file(test_directory, 'baz.txt') == baz + # Enough depth to find files. + assert common.find_file(test_directory, 'foo.txt', 2) == foo + assert common.find_file(test_directory, 'bar.txt', 2) == bar + assert common.find_file(test_directory, 'baz.txt', 2) == baz + + assert common.find_file(test_directory, 'does not exist', 100) is None diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 2c139f86..2dedde11 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -7,7 +7,7 @@ import pytest from mock import mock from wrolpi.admin import HotspotStatus -from wrolpi.common import get_config +from wrolpi.common import get_wrolpi_config from wrolpi.downloader import Download, get_download_manager_config from wrolpi.errors import ValidationError, SearchEmpty from wrolpi.root_api import json_error_handler @@ -207,7 +207,7 @@ def test_hotspot_settings(test_session, test_client, test_config): """ The User can toggle the Hotspot via /settings. The Hotspot can be automatically started on startup. """ - config = get_config() + config = get_wrolpi_config() assert config.hotspot_on_startup is True with mock.patch('wrolpi.root_api.admin') as mock_admin: @@ -598,3 +598,34 @@ def test_recursive_errors(): }, }, } + + +@pytest.mark.asyncio +async def test_settings_special_directories(test_async_client, test_config): + """Maintainer can change special directories.""" + assert get_wrolpi_config().archive_directory == 'archive' + + data = {'archive_directory': 'archives'} + request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + + assert get_wrolpi_config().archive_directory == 'archives' + + data = {'archive_directory': '/absolute/not/allowed'} + request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST + data = {'videos_directory': '/absolute/not/allowed'} + request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST + data = {'map_directory': '/absolute/not/allowed'} + request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST + data = {'zims_directory': '/absolute/not/allowed'} + request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST + + # Empty directory restores default. + data = {'archive_directory': ''} + request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + assert get_wrolpi_config().archive_directory == 'archive'