rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Allowing maintainer to change special directories.

Browse source
This commit is contained in:
Roland 2024-03-09 16:27:34 -07:00
parent 00fe799b8f
commit b05b2e7a6d
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
17 changed files with 499 additions and 195 deletions
Download patch file Download diff file Expand all files Collapse all files

348
app/src/components/admin/Settings.js
View file

@ -1,11 +1,26 @@
import React from "react";
import {getSettings, postRestart, postShutdown, saveSettings} from "../../api";
import {Button, Form, FormGroup, FormInput, Header, Loader, Modal, ModalContent, ModalHeader, Segment} from "../Theme";
import {Container, Dimmer, Icon} from "semantic-ui-react";
import {
Button,
Divider,
Form,
FormField,
FormGroup,
FormInput,
Header,
Loader,
Modal,
ModalContent,
ModalHeader,
Segment
} from "../Theme";
import {Container, Dimmer, GridColumn, GridRow, Icon, Input} from "semantic-ui-react";
import {APIButton, ErrorMessage, HelpPopup, HotspotToggle, ThrottleToggle, Toggle, WROLModeMessage} from "../Common";
import QRCode from "react-qr-code";
import {useDockerized} from "../../hooks/customHooks";
import {toast} from "react-semantic-toasts-2";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {SettingsContext} from "../../contexts/contexts";
export function ShutdownButton() {
const dockerized = useDockerized();
@ -151,11 +166,13 @@ export class SettingsPage extends React.Component {
this.state = {
disabled: false,
hotspot_encryption: 'WPA',
edit_special_directories: false,
pending: false,
qrCodeValue: '',
qrOpen: false,
ready: false,
archive_directory: null,
download_on_startup: null,
download_timeout: null,
hotspot_device: null,
@ -164,8 +181,11 @@ export class SettingsPage extends React.Component {
hotspot_status: null,
ignore_outdated_zims: null,
log_level: null,
map_directory: null,
throttle_on_startup: null,
throttle_status: null,
videos_directory: null,
zims_directory: null,
}
this.handleSubmit = this.handleSubmit.bind(this);
@ -177,6 +197,7 @@ export class SettingsPage extends React.Component {
const settings = await getSettings();
this.setState({
ready: true,
archive_directory: settings.archive_directory,
disabled: settings.wrol_mode,
download_on_startup: settings.download_on_startup,
download_timeout: settings.download_timeout || '',
@ -186,8 +207,11 @@ export class SettingsPage extends React.Component {
hotspot_status: settings.hotspot_status,
ignore_outdated_zims: settings.ignore_outdated_zims,
log_level: fromApiLogLevel(settings.log_level),
map_directory: settings.map_directory,
throttle_on_startup: settings.throttle_on_startup,
throttle_status: settings.throttle_status,
videos_directory: settings.videos_directory,
zims_directory: settings.zims_directory,
}, this.handleHotspotChange);
} catch (e) {
console.error(e);
@ -201,6 +225,7 @@ export class SettingsPage extends React.Component {
}
this.setState({disabled: true, pending: true});
let settings = {
archive_directory: this.state.archive_directory,
download_on_startup: this.state.download_on_startup,
download_timeout: this.state.download_timeout ? parseInt(this.state.download_timeout) : 0,
hotspot_device: this.state.hotspot_device,
@ -208,7 +233,10 @@ export class SettingsPage extends React.Component {
hotspot_ssid: this.state.hotspot_ssid,
ignore_outdated_zims: this.state.ignore_outdated_zims,
log_level: toApiLogLevel(this.state.log_level),
map_directory: this.state.map_directory,
throttle_on_startup: this.state.throttle_on_startup,
videos_directory: this.state.videos_directory,
zims_directory: this.state.zims_directory,
}
try {
const response = await saveSettings(settings);
@ -263,157 +291,231 @@ export class SettingsPage extends React.Component {
return <ErrorMessage>Unable to fetch settings</ErrorMessage>
}
let {
const {
archive_directory,
disabled,
download_on_startup,
download_timeout,
edit_special_directories,
hotspot_device,
hotspot_password,
hotspot_ssid,
ignore_outdated_zims,
log_level,
map_directory,
pending,
qrCodeValue,
throttle_on_startup,
videos_directory,
zims_directory,
} = this.state;
const qrButton = <Button icon style={{marginBottom: '1em'}}><Icon name='qrcode' size='big'/></Button>;
return <Container fluid>
<WROLModeMessage content='Settings are disabled because WROL Mode is enabled.'/>
return <SettingsContext.Consumer>{({settings}) => {
const mediaDirectoryLabel = `${settings.media_directory}/`;
return <Container fluid>
<WROLModeMessage content='Settings are disabled because WROL Mode is enabled.'/>
<Segment>
<HotspotToggle/>
<ThrottleToggle/>
</Segment>
<Segment>
<HotspotToggle/>
<ThrottleToggle/>
</Segment>
<Segment>
<Header as='h2'>
Settings
</Header>
<p>Any changes will be written to <i>config/wrolpi.yaml</i>.</p>
<Segment>
<Header as='h2'>Settings</Header>
<p>Any changes will be written to <i>{settings.media_directory}/config/wrolpi.yaml</i>.</p>
<Form id="settings" onSubmit={this.handleSubmit}>
<div style={{margin: '0.5em'}}>
<Toggle
label='Download on Startup'
disabled={disabled || download_on_startup === null}
checked={download_on_startup === true}
onChange={checked => this.handleInputChange(null, 'download_on_startup', checked)}
/>
</div>
<Form id="settings" onSubmit={this.handleSubmit}>
<div style={{margin: '0.5em'}}>
<Toggle
label='Download on Startup'
disabled={disabled || download_on_startup === null}
checked={download_on_startup === true}
onChange={checked => this.handleInputChange(null, 'download_on_startup', checked)}
/>
</div>
<div style={{margin: '0.5em'}}>
<Toggle
label='CPU Power-save on Startup'
disabled={disabled || throttle_on_startup === null}
checked={throttle_on_startup === true}
onChange={checked => this.handleInputChange(null, 'throttle_on_startup', checked)}
/>
</div>
<div style={{margin: '0.5em'}}>
<Toggle
label='CPU Power-save on Startup'
disabled={disabled || throttle_on_startup === null}
checked={throttle_on_startup === true}
onChange={checked => this.handleInputChange(null, 'throttle_on_startup', checked)}
/>
</div>
<div style={{margin: '0.5em'}}>
<Toggle
label='Ignore outdated Zims'
disabled={disabled || ignore_outdated_zims === null}
checked={ignore_outdated_zims === true}
onChange={checked => this.handleInputChange(null, 'ignore_outdated_zims', checked)}
/>
</div>
<div style={{margin: '0.5em'}}>
<Toggle
label='Ignore outdated Zims'
disabled={disabled || ignore_outdated_zims === null}
checked={ignore_outdated_zims === true}
onChange={checked => this.handleInputChange(null, 'ignore_outdated_zims', checked)}
/>
</div>
<FormGroup inline>
<FormInput
label={<>
<b>Download Timeout</b>
<HelpPopup content='Downloads will be stopped after this many seconds have elapsed.
<FormGroup inline>
<FormInput
label={<>
<b>Download Timeout</b>
<HelpPopup content='Downloads will be stopped after this many seconds have elapsed.
Downloads will never timeout if this is empty.'/>
</>}
value={download_timeout}
disabled={disabled || download_timeout === null}
onChange={(e, d) => this.handleTimeoutChange(e, 'download_timeout', d.value)}
</>}
value={download_timeout}
disabled={disabled || download_timeout === null}
onChange={(e, d) => this.handleTimeoutChange(e, 'download_timeout', d.value)}
/>
</FormGroup>
<FormGroup inline>
<FormInput
label='Hotspot SSID'
value={hotspot_ssid}
disabled={disabled || hotspot_ssid === null}
onChange={(e, d) => this.setState({hotspot_ssid: d.value}, this.handleHotspotChange)}
/>
<FormInput
label='Hotspot Password'
disabled={disabled || hotspot_password === null}
value={hotspot_password}
onChange={(e, d) => this.setState({hotspot_password: d.value}, this.handleHotspotChange)}
/>
<FormInput
label='Hotspot Device'
disabled={disabled || hotspot_password === null}
value={hotspot_device}
onChange={(e, d) => this.handleInputChange(e, 'hotspot_device', d.value)}
/>
</FormGroup>
<Modal closeIcon
onClose={() => this.setState({qrOpen: false})}
onOpen={this.handleQrOpen}
open={this.state.qrOpen}
trigger={qrButton}
>
<ModalHeader>
Scan this code to join the hotspot
</ModalHeader>
<ModalContent>
<div style={{display: 'inline-block', backgroundColor: '#ffffff', padding: '1em'}}>
<QRCode value={qrCodeValue} size={300}/>
</div>
</ModalContent>
</Modal>
<br/>
<label htmlFor='log_levels_input'>Log Level: {logLevelToName(log_level)}</label>
<br/>
<input type='range'
id='log_levels_input'
list='log_levels'
min='1'
max='5'
value={log_level}
onChange={e => this.setState({'log_level': parseInt(e.target.value)})}
style={{marginBottom: '1em'}}
/>
</FormGroup>
<datalist id='log_levels'>
<option value='1'>Critical</option>
<option value='2'>Warning</option>
<option value='3'>Info</option>
<option value='4'>Debug</option>
<option value='5'>All</option>
</datalist>
<FormGroup inline>
<FormInput
label='Hotspot SSID'
value={hotspot_ssid}
disabled={disabled || hotspot_ssid === null}
onChange={(e, d) => this.setState({hotspot_ssid: d.value}, this.handleHotspotChange)}
/>
<FormInput
label='Hotspot Password'
disabled={disabled || hotspot_password === null}
value={hotspot_password}
onChange={(e, d) => this.setState({hotspot_password: d.value}, this.handleHotspotChange)}
/>
<FormInput
label='Hotspot Device'
disabled={disabled || hotspot_password === null}
value={hotspot_device}
onChange={(e, d) => this.handleInputChange(e, 'hotspot_device', d.value)}
/>
</FormGroup>
<Divider/>
<Modal closeIcon
onClose={() => this.setState({qrOpen: false})}
onOpen={this.handleQrOpen}
open={this.state.qrOpen}
trigger={qrButton}
>
<ModalHeader>
Scan this code to join the hotspot
</ModalHeader>
<ModalContent>
<div style={{display: 'inline-block', backgroundColor: '#ffffff', padding: '1em'}}>
<QRCode value={qrCodeValue} size={300}/>
</div>
</ModalContent>
</Modal>
<Header as='h3'>Special Directories</Header>
<p>WROLPi will save files to these directories (any video files will be saved to
the <i>videos</i> by default, etc.).</p>
<br/>
<Grid stackable>
<GridRow columns={2}>
<GridColumn>
<FormField>
<label>Archive Directory</label>
<Input
label={mediaDirectoryLabel}
value={archive_directory}
disabled={!edit_special_directories}
onChange={(e, d) => this.handleInputChange(e, 'archive_directory', d.value)}
/>
</FormField>
</GridColumn>
<GridColumn>
<FormField>
<label>Videos Directory</label>
<Input
label={mediaDirectoryLabel}
value={videos_directory}
disabled={!edit_special_directories}
onChange={(e, d) => this.handleInputChange(e, 'videos_directory', d.value)}
/>
</FormField>
</GridColumn>
</GridRow>
<GridRow columns={2}>
<GridColumn>
<FormField>
<label>Map Directory</label>
<Input
label={mediaDirectoryLabel}
value={map_directory}
disabled={!edit_special_directories}
onChange={(e, d) => this.handleInputChange(e, 'map_directory', d.value)}
/>
</FormField>
</GridColumn>
<GridColumn>
<FormField>
<label>Zims Directory</label>
<Input
label={mediaDirectoryLabel}
value={zims_directory}
disabled={!edit_special_directories}
onChange={(e, d) => this.handleInputChange(e, 'zims_directory', d.value)}
/>
</FormField>
</GridColumn>
</GridRow>
<GridRow columns={1}>
<GridColumn>
<Toggle
label='Edit Directories'
disabled={disabled}
checked={edit_special_directories === true}
onChange={checked => this.handleInputChange(null, 'edit_special_directories', checked)}
/>
</GridColumn>
</GridRow>
</Grid>
<label htmlFor='log_levels_input'>Log Level: {logLevelToName(log_level)}</label>
<br/>
<input type='range'
id='log_levels_input'
list='log_levels'
min='1'
max='5'
value={log_level}
onChange={e => this.setState({'log_level': parseInt(e.target.value)})}
style={{marginBottom: '1em'}}
/>
<datalist id='log_levels'>
<option value='1'>Critical</option>
<option value='2'>Warning</option>
<option value='3'>Info</option>
<option value='4'>Debug</option>
<option value='5'>All</option>
</datalist>
<Divider/>
<br/>
<APIButton
color='violet'
size='big'
onClick={this.handleSubmit}
obeyWROLMode={true}
disabled={disabled}
>Save</APIButton>
<APIButton
color='violet'
size='big'
onClick={this.handleSubmit}
obeyWROLMode={true}
disabled={disabled}
>Save</APIButton>
<Dimmer active={pending}>
<Loader active={pending} size='large'/>
</Dimmer>
<Dimmer active={pending}>
<Loader active={pending} size='large'/>
</Dimmer>
</Form>
</Segment>
</Form>
</Segment>
<Segment>
<Header as='h3'>Control WROLPi</Header>
<RestartButton/>
<ShutdownButton/>
</Segment>
</Container>
<Segment>
<Header as='h3'>Control WROLPi</Header>
<RestartButton/>
<ShutdownButton/>
</Segment>
</Container>;
}
}</SettingsContext.Consumer>
}
}

6
main.py
View file

@ -9,7 +9,7 @@ from sanic.signals import Event
from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS
from wrolpi import root_api, admin
from wrolpi.common import logger, get_config, check_media_directory, limit_concurrent, \
from wrolpi.common import logger, get_wrolpi_config, check_media_directory, limit_concurrent, \
wrol_mode_enabled, cancel_refresh_tasks, set_log_level, background_task, cancel_background_tasks
from wrolpi.dates import Seconds
from wrolpi.downloader import download_manager, import_downloads_config
@ -136,7 +136,7 @@ def main():
if args.sub_commands == 'db':
return db_main(args)
config = get_config()
config = get_wrolpi_config()
# Hotspot/throttle are not supported in Docker containers.
if not DOCKERIZED and config.hotspot_on_startup:
@ -246,7 +246,7 @@ async def periodic_downloads(app: Sanic):
download_manager.disable()
return
config = get_config()
config = get_wrolpi_config()
if config.download_on_startup is False:
logger.warning('Not starting download manager because Downloads are disabled on startup.')
download_manager.disable()

4
modules/archive/lib.py
View file

@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from wrolpi import dates
from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
@ -48,7 +48,7 @@ class ArchiveFiles:
def get_archive_directory() -> pathlib.Path:
archive_directory = get_media_directory() / 'archive'
archive_directory = get_media_directory() / get_wrolpi_config().archive_directory
return archive_directory

12
modules/archive/test/test_lib.py
View file

@ -13,6 +13,7 @@ from modules.archive import lib
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, model_archive_result, \
get_domains
from modules.archive.models import Archive, Domain
from wrolpi.common import get_wrolpi_config
from wrolpi.db import get_db_session
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
@ -817,3 +818,14 @@ SINGLEFILE_EXAMPLE_1 = b'''<!DOCTYPE html>
def test_get_url_from_singlefile():
assert lib.get_url_from_singlefile(SINGLEFILE_EXAMPLE_1) == 'https://www.example.com'
def test_get_custom_archive_directory(test_directory, test_config):
"""Custom directory can be used for archive directory."""
# Default location.
assert lib.get_archive_directory() == (test_directory / 'archive')
get_wrolpi_config().archive_directory = 'custom/archives'
assert lib.get_archive_directory() == (test_directory / 'custom/archives')
assert lib.get_domain_directory('https://example.com') == (test_directory / 'custom/archives/example.com')

8
modules/map/lib.py
View file

@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
from modules.map.models import MapFile
from wrolpi import flags
from wrolpi.cmd import SUDO_BIN
from wrolpi.common import get_media_directory, walk, logger
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config
from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp
from wrolpi.db import optional_session, get_db_session
from wrolpi.events import Events
@ -23,14 +23,14 @@ IMPORTING = Manager().dict(dict(
def get_map_directory() -> Path:
map_directory = get_media_directory() / 'map'
map_directory = get_media_directory() / get_wrolpi_config().map_directory
if not map_directory.is_dir():
map_directory.mkdir()
map_directory.mkdir(parents=True)
return map_directory
def is_pbf_file(pbf: Path) -> bool:
"""Uses file command to check type of a file. Returns True if a file is an OpenStreetMap PBF file."""
"""Uses file command to check type of file. Returns True if a file is an OpenStreetMap PBF file."""
cmd = ('/usr/bin/file', pbf)
try:
output = subprocess.check_output(cmd)

12
modules/map/test/test_lib.py
View file

@ -4,6 +4,7 @@ from unittest import mock
import pytest
from modules.map import lib
from wrolpi.common import get_wrolpi_config
def test_get_map_paths(test_directory, make_files_structure):
@ -101,3 +102,14 @@ def test_seconds_to_import_rpi4(size, expected):
def test_seconds_to_import_rpi5(size, expected):
with unittest.mock.patch('modules.map.lib.IS_RPI5', True):
assert lib.seconds_to_import(size) == expected
def test_get_custom_map_directory(test_directory, test_config):
"""Custom directory can be used for map directory."""
# Default location.
assert lib.get_map_directory() == (test_directory / 'map')
get_wrolpi_config().map_directory = 'custom/deep/map/directory'
assert lib.get_map_directory() == (test_directory / 'custom/deep/map/directory')
assert (test_directory / 'custom/deep/map/directory').is_dir()

5
modules/videos/common.py
View file

@ -17,7 +17,7 @@ from sqlalchemy.orm import Session
from wrolpi.captions import FFMPEG_BIN
from wrolpi.cmd import FFPROBE_BIN
from wrolpi.common import logger, get_media_directory
from wrolpi.common import logger, get_media_directory, get_wrolpi_config
from wrolpi.dates import seconds_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
@ -35,7 +35,8 @@ class ConfigError(Exception):
def get_videos_directory() -> pathlib.Path:
"""Get the "videos" directory in the media directory. Make it if it does not exist."""
directory = get_media_directory() / 'videos'
config = get_wrolpi_config()
directory = get_media_directory() / config.videos_directory
if not directory.is_dir():
directory.mkdir(parents=True)
return directory

22
modules/videos/test/test_common.py
View file

@ -7,7 +7,7 @@ import pytest
from PIL import Image
from modules.videos.models import Channel, Video
from wrolpi.common import get_absolute_media_path
from wrolpi.common import get_absolute_media_path, get_wrolpi_config
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.files import lib as files_lib
from wrolpi.vars import PROJECT_DIR
@ -273,3 +273,23 @@ async def test_video_ffprobe_json(test_session, video_file):
video = test_session.query(Video).one()
assert video.ffprobe_json
assert video.file_group.length
def test_get_videos_directory(test_directory):
"""Default videos directory path"""
# Directory does not yet exist.
assert not (test_directory / 'videos').exists()
# Directory is created when first gotten.
assert common.get_videos_directory() == (test_directory / 'videos')
assert (test_directory / 'videos').is_dir()
def test_get_custom_videos_directory(test_directory, test_config):
"""Custom directory can be used for videos directory."""
config = get_wrolpi_config()
config.videos_directory = 'custom/directory/videos'
assert common.get_videos_directory() == (test_directory / 'custom/directory/videos')
assert (test_directory / 'custom/directory/videos').is_dir()
assert not (test_directory / 'videos').is_dir()

5
modules/zim/lib.py
View file

@ -16,7 +16,7 @@ from modules.zim.kiwix import KIWIX_CATALOG
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
from wrolpi import flags
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
register_refresh_cleanup, background_task
register_refresh_cleanup, background_task, get_wrolpi_config
from wrolpi.db import get_db_session, optional_session, get_db_curs
from wrolpi.downloader import DownloadFrequency
from wrolpi.files.lib import refresh_files, split_file_name_words
@ -464,8 +464,7 @@ def zim_download_url_to_name(url: str) -> Tuple[str, str]:
def get_zim_directory() -> pathlib.Path:
media_directory = get_media_directory()
zim_directory = media_directory / 'zims'
zim_directory = get_media_directory() / get_wrolpi_config().zims_directory
return zim_directory

11
modules/zim/test/test_lib.py
View file

@ -8,7 +8,7 @@ from modules.zim import lib
from modules.zim.errors import UnknownZim
from modules.zim.models import TagZimEntry, ZimSubscription
from wrolpi import tags, flags
from wrolpi.common import DownloadFileInfo
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
from wrolpi.downloader import Download, import_downloads_config, save_downloads_config
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
@ -341,3 +341,12 @@ async def test_zim_modeler(test_session, zim_factory):
fg: FileGroup = test_session.query(FileGroup).one()
assert fg.title == 'the_zim_title'
assert fg.a_text == 'the zim title'
def test_get_custom_zims_directory(test_directory, test_config):
"""Custom directory can be used for zims directory."""
assert lib.get_zim_directory() == (test_directory / 'zims')
get_wrolpi_config().zims_directory = 'custom/zims'
assert lib.get_zim_directory() == (test_directory / 'custom/zims')

148
wrolpi/common.py
View file

@ -126,7 +126,7 @@ __all__ = [
'extract_headlines',
'extract_html_text',
'get_absolute_media_path',
'get_config',
'get_wrolpi_config',
'get_download_info',
'get_files_and_directories',
'get_html_soup',
@ -212,6 +212,51 @@ def compile_tsvector(element, compiler, **kw):
URL_CHARS = string.ascii_lowercase + string.digits
def find_file(directory: pathlib.Path, name: str, depth=1) -> Optional[pathlib.Path]:
"""Recursively searches a directory for a file with the provided name."""
if depth == 0:
return
for path in sorted(directory.iterdir()):
if path.is_file() and path.name == name:
return path
if path.is_dir() and (result := find_file(path, name, depth - 1)):
return result
TEST_MEDIA_DIRECTORY = None
def set_test_media_directory(path):
global TEST_MEDIA_DIRECTORY
if isinstance(path, pathlib.Path):
TEST_MEDIA_DIRECTORY = path
elif path:
TEST_MEDIA_DIRECTORY = pathlib.Path(path)
else:
TEST_MEDIA_DIRECTORY = None
def get_media_directory() -> Path:
"""Get the media directory.
This will typically be /opt/wrolpi, or something else from the .env.
During testing, this function returns TEST_MEDIA_DIRECTORY.
"""
global TEST_MEDIA_DIRECTORY
if PYTEST and not TEST_MEDIA_DIRECTORY:
raise ValueError('No test media directory set during testing!!')
if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path):
if not str(TEST_MEDIA_DIRECTORY).startswith('/tmp'):
raise RuntimeError('Refusing to run test outside tmp directory!')
return TEST_MEDIA_DIRECTORY
return MEDIA_DIRECTORY
class ConfigFile:
"""
This class keeps track of the contents of a config file. You can update the config (and it's file) by
@ -306,6 +351,7 @@ class ConfigFile:
class WROLPiConfig(ConfigFile):
file_name = 'wrolpi.yaml'
default_config = dict(
archive_directory='archive',
download_on_startup=True,
download_timeout=0,
hotspot_device='wlan0',
@ -314,10 +360,34 @@ class WROLPiConfig(ConfigFile):
hotspot_ssid='WROLPi',
ignore_outdated_zims=False,
ignored_directories=list(),
map_directory='map',
throttle_on_startup=False,
videos_directory='videos',
wrol_mode=False,
zims_directory='zims',
)
def get_file(self) -> Path:
"""WROLPiConfig must be discovered so that other config files can be found."""
if not self.file_name:
raise NotImplementedError(f'You must define a file name for this {self.__class__.__name__} config.')
if PYTEST:
return get_media_directory() / f'config/{self.file_name}'
# Use the usual "/media/wrolpi/config/wrolpi.yaml" if it exists.
default_config_path = CONFIG_DIR / self.file_name
if default_config_path.is_file():
return default_config_path
# Search the media directory for the special "wrolpi.yaml" file. Assume the first one found is the config.
# Use a depth of 3; multiple drives may exist and the config directory may be down in a second drive.
if config_path := find_file(get_media_directory(), self.file_name, 3):
return config_path
# Not testing, and can't find file deep in media directory. Use the default.
return default_config_path
@property
def download_on_startup(self) -> bool:
return self._config['download_on_startup']
@ -398,12 +468,44 @@ class WROLPiConfig(ConfigFile):
def ignored_directories(self, value: List[str]):
self.update({'ignored_directories': value})
@property
def videos_directory(self) -> str:
return self._config['videos_directory']
@videos_directory.setter
def videos_directory(self, value: str):
self.update({'videos_directory': value})
@property
def archive_directory(self) -> str:
return self._config['archive_directory']
@archive_directory.setter
def archive_directory(self, value: str):
self.update({'archive_directory': value})
@property
def map_directory(self) -> str:
return self._config['map_directory']
@map_directory.setter
def map_directory(self, value: str):
self.update({'map_directory': value})
@property
def zims_directory(self) -> str:
return self._config['zims_directory']
@zims_directory.setter
def zims_directory(self, value: str):
self.update({'zims_directory': value})
WROLPI_CONFIG: WROLPiConfig = WROLPiConfig()
TEST_WROLPI_CONFIG: WROLPiConfig = None
def get_config() -> WROLPiConfig:
def get_wrolpi_config() -> WROLPiConfig:
"""Read the global WROLPi yaml config file."""
global TEST_WROLPI_CONFIG
if isinstance(TEST_WROLPI_CONFIG, WROLPiConfig):
@ -420,7 +522,7 @@ def set_test_config(enable: bool):
def wrol_mode_enabled() -> bool:
"""Return True if WROL Mode is enabled."""
return get_config().wrol_mode
return get_wrolpi_config().wrol_mode
def wrol_mode_check(func):
@ -445,7 +547,7 @@ def enable_wrol_mode(download_manager=None):
Stop downloads and Download Manager.
"""
logger_.warning('ENABLING WROL MODE')
get_config().wrol_mode = True
get_wrolpi_config().wrol_mode = True
if not download_manager:
from wrolpi.downloader import download_manager
download_manager.stop()
@ -461,7 +563,7 @@ def disable_wrol_mode(download_manager=None):
Start downloads and Download Manager.
"""
logger_.warning('DISABLING WROL MODE')
get_config().wrol_mode = False
get_wrolpi_config().wrol_mode = False
if not download_manager:
from wrolpi.downloader import download_manager
download_manager.enable()
@ -549,39 +651,6 @@ def run_after(after: callable, *args, **kwargs) -> callable:
return wrapper
TEST_MEDIA_DIRECTORY = None
def set_test_media_directory(path):
global TEST_MEDIA_DIRECTORY
if isinstance(path, pathlib.Path):
TEST_MEDIA_DIRECTORY = path
elif path:
TEST_MEDIA_DIRECTORY = pathlib.Path(path)
else:
TEST_MEDIA_DIRECTORY = None
def get_media_directory() -> Path:
"""Get the media directory.
This will typically be /opt/wrolpi, or something else from the .env.
During testing, this function returns TEST_MEDIA_DIRECTORY.
"""
global TEST_MEDIA_DIRECTORY
if PYTEST and not TEST_MEDIA_DIRECTORY:
raise ValueError('No test media directory set during testing!!')
if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path):
if not str(TEST_MEDIA_DIRECTORY).startswith('/tmp'):
raise RuntimeError('Refusing to run test outside tmp directory!')
return TEST_MEDIA_DIRECTORY
return MEDIA_DIRECTORY
MEDIA_DIRECTORY_PERMISSIONS = 0o40755
@ -1190,7 +1259,8 @@ def timer(name, level: str = 'debug'):
try:
yield
finally:
log_method(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
elapsed = (datetime.now() - before).total_seconds()
log_method(f'{name} elapsed {elapsed} seconds')
def async_timer(coro: Coroutine, name: str = 'async timer', level: str = 'debug') -> callable:

10
wrolpi/files/lib.py
View file

@ -23,7 +23,7 @@ from wrolpi import flags
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
partition, cancelable_wrapper, \
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, get_config, \
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, get_wrolpi_config, \
timer
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
@ -435,7 +435,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]:
ignored_directories = list(map(str, get_config().ignored_directories))
ignored_directories = list(map(str, get_wrolpi_config().ignored_directories))
files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)]
return files
@ -1323,7 +1323,7 @@ def add_ignore_directory(directory: Union[pathlib.Path, str]):
if not str(directory).startswith(str(media_directory)):
raise InvalidDirectory('Cannot ignore directory not in media directory')
wrolpi_config = get_config()
wrolpi_config = get_wrolpi_config()
ignored_directories = wrolpi_config.ignored_directories if wrolpi_config.ignored_directories else list()
if str(directory) in ignored_directories:
@ -1337,9 +1337,9 @@ def add_ignore_directory(directory: Union[pathlib.Path, str]):
def remove_ignored_directory(directory: Union[pathlib.Path, str]):
"""Remove a directory from the `ignored_directories` in the WROLPi config."""
directory = str(pathlib.Path(directory))
ignored_directories = get_config().ignored_directories
ignored_directories = get_wrolpi_config().ignored_directories
if directory in ignored_directories:
ignored_directories.remove(directory)
get_config().ignored_directories = ignored_directories
get_wrolpi_config().ignored_directories = ignored_directories
else:
raise UnknownDirectory('Directory is not ignored')

10
wrolpi/files/test/test_api.py
View file

@ -4,7 +4,7 @@ from unittest import mock
import pytest
from wrolpi.common import get_config
from wrolpi.common import get_wrolpi_config
from wrolpi.files import lib
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile
@ -634,13 +634,13 @@ async def test_get_file(test_session, test_async_client, test_directory, make_fi
async def test_ignore_directory(test_session, test_async_client, test_directory, make_files_structure, test_config):
"""A maintainer can ignore/un-ignore directories. The files in the directory should not be refreshed."""
foo, bar, baz = make_files_structure(['foo/foo.txt', 'foo/bar.txt', 'baz/baz.txt'])
assert len(get_config().ignored_directories) == 0
assert len(get_wrolpi_config().ignored_directories) == 0
# Ignore baz/
content = dict(path=str(baz.parent))
request, response = await test_async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.OK
assert len(get_config().ignored_directories) == 1
assert len(get_wrolpi_config().ignored_directories) == 1
await lib.refresh_files()
@ -650,7 +650,7 @@ async def test_ignore_directory(test_session, test_async_client, test_directory,
# Un-ignore baz/
request, response = await test_async_client.post('/api/files/unignore_directory', json=content)
assert response.status_code == HTTPStatus.OK
assert len(get_config().ignored_directories) == 0
assert len(get_wrolpi_config().ignored_directories) == 0
await lib.refresh_files()
@ -661,4 +661,4 @@ async def test_ignore_directory(test_session, test_async_client, test_directory,
content = dict(path=str(test_directory / 'videos'))
request, response = await test_async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert len(get_config().ignored_directories) == 0
assert len(get_wrolpi_config().ignored_directories) == 0

35
wrolpi/root_api.py
View file

@ -1,5 +1,6 @@
import asyncio
import json
import pathlib
import re
from datetime import datetime, date, timezone
from decimal import Decimal
@ -21,7 +22,7 @@ from vininfo.details._base import VinDetails
from wrolpi import admin, status, flags, schema, dates
from wrolpi import tags
from wrolpi.admin import HotspotStatus
from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \
from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, Base, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, LOG_LEVEL, \
set_global_log_level, get_relative_to_media_directory
from wrolpi.dates import now
@ -139,11 +140,12 @@ async def echo(request: Request):
@openapi.description('Get WROLPi settings')
@openapi.response(HTTPStatus.OK, schema.SettingsResponse)
def get_settings(_: Request):
config = get_config()
config = get_wrolpi_config()
ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories]
settings = {
'archive_directory': config.archive_directory,
'download_manager_disabled': download_manager.disabled.is_set(),
'download_manager_stopped': download_manager.stopped.is_set(),
'download_on_startup': config.download_on_startup,
@ -153,14 +155,17 @@ def get_settings(_: Request):
'hotspot_password': config.hotspot_password,
'hotspot_ssid': config.hotspot_ssid,
'hotspot_status': admin.hotspot_status().name,
'log_level': LOG_LEVEL.value,
'ignore_outdated_zims': config.ignore_outdated_zims,
'ignored_directories': ignored_directories,
'log_level': LOG_LEVEL.value,
'map_directory': config.map_directory,
'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative.
'throttle_on_startup': config.throttle_on_startup,
'throttle_status': admin.throttle_status().name,
'version': __version__,
'videos_directory': config.videos_directory,
'wrol_mode': config.wrol_mode,
'zims_directory': config.zims_directory,
}
return json_response(settings)
@ -168,7 +173,7 @@ def get_settings(_: Request):
@api_bp.patch('/settings')
@openapi.description('Update WROLPi settings')
@validate(json=schema.SettingsRequest)
def update_settings(request: Request, body: schema.SettingsRequest):
def update_settings(_: Request, body: schema.SettingsRequest):
if wrol_mode_enabled() and body.wrol_mode is None:
# Cannot update settings while WROL Mode is enabled, unless you want to disable WROL Mode.
raise WROLModeEnabled()
@ -187,11 +192,31 @@ def update_settings(request: Request, body: schema.SettingsRequest):
# Remove any keys with None values, then save the config.
config = {k: v for k, v in body.__dict__.items() if v is not None}
wrolpi_config = get_config()
wrolpi_config = get_wrolpi_config()
if not config:
raise InvalidConfig()
if body.archive_directory and pathlib.Path(body.archive_directory).is_absolute():
raise InvalidConfig('Archive directory must be relative to media directory')
elif not body.archive_directory:
config['archive_directory'] = wrolpi_config.default_config['archive_directory']
if body.videos_directory and pathlib.Path(body.videos_directory).is_absolute():
raise InvalidConfig('Videos directory must be relative to media directory')
elif not body.videos_directory:
config['videos_directory'] = wrolpi_config.default_config['videos_directory']
if body.map_directory and pathlib.Path(body.map_directory).is_absolute():
raise InvalidConfig('Map directory must be relative to media directory')
elif not body.map_directory:
config['map_directory'] = wrolpi_config.default_config['map_directory']
if body.zims_directory and pathlib.Path(body.zims_directory).is_absolute():
raise InvalidConfig('Zims directory must be relative to media directory')
elif not body.zims_directory:
config['zims_directory'] = wrolpi_config.default_config['zims_directory']
log_level = config.pop('log_level', None)
if isinstance(log_level, int):
set_global_log_level(log_level)

4
wrolpi/schema.py
View file

@ -16,6 +16,7 @@ class SettingsResponse:
@dataclass
class SettingsRequest:
archive_directory: Optional[str] = None
download_on_startup: Optional[bool] = None
download_timeout: Optional[int] = None
hotspot_device: Optional[str] = None
@ -25,10 +26,13 @@ class SettingsRequest:
hotspot_status: Optional[bool] = None
ignore_outdated_zims: Optional[bool] = None
log_level: Optional[int] = None
map_directory: Optional[str] = None
media_directory: Optional[str] = None
throttle_on: Optional[bool] = None
throttle_on_startup: Optional[bool] = None
videos_directory: Optional[str] = None
wrol_mode: Optional[bool] = None
zims_directory: Optional[str] = None
@dataclass

19
wrolpi/test/test_common.py
View file

@ -778,3 +778,22 @@ def test_format_html_file():
)
def test_chain(iterable, length, step, expected):
assert list(common.chain(iterable, length)) == expected
def test_find_file(test_directory, make_files_structure):
bar, baz, foo = make_files_structure({
'foo/foo.txt': 'foo',
'bar/bar.txt': 'bar',
'baz.txt': 'baz',
})
# Files are too deep.
assert common.find_file(test_directory, 'foo.txt') is None
assert common.find_file(test_directory, 'bar.txt') is None
assert common.find_file(test_directory, 'baz.txt') == baz
# Enough depth to find files.
assert common.find_file(test_directory, 'foo.txt', 2) == foo
assert common.find_file(test_directory, 'bar.txt', 2) == bar
assert common.find_file(test_directory, 'baz.txt', 2) == baz
assert common.find_file(test_directory, 'does not exist', 100) is None

35
wrolpi/test/test_root_api.py
View file

@ -7,7 +7,7 @@ import pytest
from mock import mock
from wrolpi.admin import HotspotStatus
from wrolpi.common import get_config
from wrolpi.common import get_wrolpi_config
from wrolpi.downloader import Download, get_download_manager_config
from wrolpi.errors import ValidationError, SearchEmpty
from wrolpi.root_api import json_error_handler
@ -207,7 +207,7 @@ def test_hotspot_settings(test_session, test_client, test_config):
"""
The User can toggle the Hotspot via /settings. The Hotspot can be automatically started on startup.
"""
config = get_config()
config = get_wrolpi_config()
assert config.hotspot_on_startup is True
with mock.patch('wrolpi.root_api.admin') as mock_admin:
@ -598,3 +598,34 @@ def test_recursive_errors():
},
},
}
@pytest.mark.asyncio
async def test_settings_special_directories(test_async_client, test_config):
"""Maintainer can change special directories."""
assert get_wrolpi_config().archive_directory == 'archive'
data = {'archive_directory': 'archives'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
assert get_wrolpi_config().archive_directory == 'archives'
data = {'archive_directory': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {'videos_directory': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {'map_directory': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {'zims_directory': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
# Empty directory restores default.
data = {'archive_directory': ''}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
assert get_wrolpi_config().archive_directory == 'archive'