diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 693bbe0b..25b34fbb 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -16,6 +16,7 @@ from selenium import webdriver from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive +from wrolpi.cmd import which from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir, escape_file_name, walk from wrolpi.dates import now, Seconds, local_timezone from wrolpi.db import get_db_session, get_db_curs, get_ranked_models @@ -55,9 +56,7 @@ def get_archive_directory() -> pathlib.Path: def get_domain_directory(url: str) -> pathlib.Path: - """ - Get the archive directory for a particular domain. - """ + """Get the archive directory for a particular domain.""" domain = extract_domain(url) directory = get_archive_directory() / domain if directory.is_dir(): @@ -70,9 +69,7 @@ def get_domain_directory(url: str) -> pathlib.Path: def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles: - """ - Create a list of archive files using a shared name schema. Raise an error if any of them exist. - """ + """Create a list of archive files using a shared name schema. Raise an error if any of them exist.""" directory = get_domain_directory(url) # Datetime is valid in Linux and Windows. dt = archive_strftime(now()) @@ -149,15 +146,18 @@ def request_archive(url: str): return singlefile, readability, screenshot +SINGLE_FILE_PATH = which('single-file', + '/usr/bin/single-file', # rpi ubuntu + '/usr/local/bin/single-file', # debian + warn=True) + + def local_singlefile(url: str): """Run the single-file executable to create an HTML file archive.""" - single_file_path = pathlib.Path('/usr/bin/single-file') - if not single_file_path.is_file(): - single_file_path = pathlib.Path('/usr/local/bin/single-file') - if not single_file_path.is_file(): - raise FileNotFoundError(f'single-file not found at {single_file_path}') + if not SINGLE_FILE_PATH.is_file(): + raise FileNotFoundError(f'single-file not found') - cmd = (str(single_file_path), + cmd = (str(SINGLE_FILE_PATH), url, '--browser-executable-path', '/usr/bin/chromium-browser', '--browser-args', '["--no-sandbox"]', @@ -169,7 +169,7 @@ def local_singlefile(url: str): def local_screenshot(url: str) -> bytes: - """Take a screenshot of the URL using selenium.""" + """Take a screenshot of the URL using chromedriver.""" logger.info(f'Screenshot: {url}') # Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page. @@ -188,16 +188,19 @@ def local_screenshot(url: str) -> bytes: return png +READABILITY_PATH = which('readability-extractor', + '/usr/bin/readability-extractor', # rpi ubuntu + '/usr/local/bin/readability-extractor', # debian + warn=True) + + def local_extract_readability(path: str, url: str) -> dict: """Extract the readability from an HTML file, typically from single-file.""" logger.info(f'readability for {url}') - readability_path = pathlib.Path('/usr/bin/readability-extractor') - if not readability_path.is_file(): - readability_path = pathlib.Path('/usr/local/bin/readability-extractor') - if not readability_path.is_file(): - raise FileNotFoundError(f'Readability extractor not found at {readability_path}') + if not READABILITY_PATH.is_file(): + raise FileNotFoundError(f'Readability extractor not found') - cmd = (readability_path, path, url) + cmd = (READABILITY_PATH, path, url) logger.debug(f'readability cmd: {cmd}') output = subprocess.check_output(cmd, timeout=60 * 3) output = json.loads(output) diff --git a/modules/map/lib.py b/modules/map/lib.py index f9105129..fabc5df1 100644 --- a/modules/map/lib.py +++ b/modules/map/lib.py @@ -7,6 +7,7 @@ from typing import List from sqlalchemy.orm import Session from modules.map.models import MapFile +from wrolpi.cmd import which from wrolpi.common import get_media_directory, walk, logger, wrol_mode_check from wrolpi.dates import now, timedelta_to_timestamp from wrolpi.db import optional_session, get_db_session @@ -104,9 +105,12 @@ async def import_pbfs(pbfs: List[str]): IMPORT_EVENT.clear() +BASH = which('bash', '/bin/bash', warn=True) + + async def import_pbf(pbf: Path): """Run the osm2pgsql binary to import a PBF map file.""" - cmd = f'/bin/bash {PROJECT_DIR}/scripts/import_map.sh {pbf.absolute()}' + cmd = f'{BASH} {PROJECT_DIR}/scripts/import_map.sh {pbf.absolute()}' logger.debug(f'Running import script: {cmd}') start = now() proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE) diff --git a/wrolpi/admin.py b/wrolpi/admin.py index a7545800..cd857468 100644 --- a/wrolpi/admin.py +++ b/wrolpi/admin.py @@ -1,28 +1,16 @@ import enum -import pathlib import subprocess +from wrolpi.cmd import which from wrolpi.common import logger -from wrolpi.vars import PYTEST, DEFAULT_CPU_FREQUENCY, DOCKERIZED +from wrolpi.vars import DEFAULT_CPU_FREQUENCY, DOCKERIZED logger = logger.getChild(__name__) -# Location on an RPi -SUDO_BIN = pathlib.Path('/usr/bin/sudo') -if not SUDO_BIN.is_file() and not PYTEST and not DOCKERIZED: - logger.error('COULD NOT FIND sudo!!!') - -NMCLI_BIN = pathlib.Path('/usr/bin/nmcli') -if not NMCLI_BIN.is_file() and not PYTEST and not DOCKERIZED: - logger.error('COULD NOT FIND nmcli!!!') - -CPUFREQ_INFO_BIN = pathlib.Path('/usr/bin/cpufreq-info') -if not CPUFREQ_INFO_BIN.is_file() and not PYTEST and not DOCKERIZED: - logger.error('COULD NOT FIND cpufreq-info!!!') - -CPUFREQ_SET_BIN = pathlib.Path('/usr/bin/cpufreq-set') -if not CPUFREQ_SET_BIN.is_file() and not PYTEST and not DOCKERIZED: - logger.error('COULD NOT FIND cpufreq-set!!!') +SUDO = which('sudo', '/usr/bin/sudo') +NMCLI = which('nmcli', '/usr/bin/nmcli') +CPUFREQ_INFO = which('cpufreq-info', '/usr/bin/cpufreq-info') +CPUFREQ_SET = which('cpufreq-set', '/usr/bin/cpufreq-set') POWER_SAVE_FREQ = 'powersave' # noqa @@ -35,7 +23,7 @@ class HotspotStatus(enum.Enum): def hotspot_status() -> HotspotStatus: - cmd = (NMCLI_BIN,) + cmd = (NMCLI,) try: output = subprocess.check_output(cmd).decode().strip() except FileNotFoundError as e: @@ -67,13 +55,13 @@ def enable_hotspot(): return enable_hotspot() elif status == HotspotStatus.disconnected: # Radio is on, but not connected. Good, turn it into a hotspot. - cmd = (SUDO_BIN, NMCLI_BIN, 'device', 'wifi', 'hotspot', 'ifname', 'wlan0', + cmd = (SUDO, NMCLI, 'device', 'wifi', 'hotspot', 'ifname', 'wlan0', 'ssid', 'WROLPi', 'password', 'wrolpi hotspot') subprocess.check_call(cmd) return True elif status == HotspotStatus.unavailable: # Radio is not on, turn it on. - cmd = (SUDO_BIN, NMCLI_BIN, 'radio', 'wifi', 'on') + cmd = (SUDO, NMCLI, 'radio', 'wifi', 'on') subprocess.check_call(cmd) return enable_hotspot() elif status == HotspotStatus.unknown: @@ -82,7 +70,7 @@ def enable_hotspot(): def disable_hotspot(): """Turn off the wlan0 interface.""" - cmd = (SUDO_BIN, NMCLI_BIN, 'radio', 'wifi', 'off') + cmd = (SUDO, NMCLI, 'radio', 'wifi', 'off') try: subprocess.check_call(cmd) return True @@ -109,7 +97,7 @@ GOVERNOR_MAP = { def throttle_status() -> GovernorStatus: - cmd = (CPUFREQ_INFO_BIN,) + cmd = (CPUFREQ_INFO,) try: output = subprocess.check_output(cmd) except FileNotFoundError as e: @@ -129,7 +117,7 @@ def throttle_status() -> GovernorStatus: def throttle_cpu_on() -> bool: """Call cpufreq-set to throttle CPU.""" try: - cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', POWER_SAVE_FREQ) + cmd = (SUDO, CPUFREQ_SET, '-g', POWER_SAVE_FREQ) subprocess.check_call(cmd) return True except FileNotFoundError: @@ -140,7 +128,7 @@ def throttle_cpu_on() -> bool: def throttle_cpu_off() -> bool: """Call cpufreq-set to un-throttle CPU.""" try: - cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', DEFAULT_CPU_FREQUENCY) + cmd = (SUDO, CPUFREQ_SET, '-g', DEFAULT_CPU_FREQUENCY) subprocess.check_call(cmd) return True except FileNotFoundError: diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py new file mode 100644 index 00000000..18fb42d3 --- /dev/null +++ b/wrolpi/cmd.py @@ -0,0 +1,33 @@ +import os +import shutil +from pathlib import Path + +from wrolpi.common import logger +from wrolpi.vars import PYTEST, DOCKERIZED + +logger = logger.getChild(__name__) + + +def is_executable(path: Path) -> bool: + """Returns True if the file is executable.""" + return path.is_file() and os.access(path, os.X_OK) + + +def which(name: str, *possible_paths: str, warn: bool = False) -> Path: + """ + Find an executable in the system $PATH. If the executable cannot be found in $PATH, then return the first + executable found in possible_paths. + + Returns None if no executable can be found. + """ + path = shutil.which(name) + if path: + return Path(path).absolute() + + possible_paths = map(Path, possible_paths) + for path in possible_paths: + if is_executable(path): + return path.absolute() + + if warn and not PYTEST and not DOCKERIZED: + logger.warning(f'Cannot find executable {name}') diff --git a/wrolpi/test/test_admin.py b/wrolpi/test/test_admin.py index 9055e2ae..8a38aed2 100644 --- a/wrolpi/test/test_admin.py +++ b/wrolpi/test/test_admin.py @@ -64,7 +64,8 @@ def test_enable_hotspot_connected(): def test_throttle_on(): - with mock.patch('wrolpi.admin.subprocess') as mock_subprocess: + with mock.patch('wrolpi.admin.subprocess') as mock_subprocess, \ + mock.patch('wrolpi.admin.CPUFREQ_SET', PosixPath('/usr/bin/cpufreq-set')): assert admin.throttle_cpu_on() mock_subprocess.check_call.assert_called_once_with( (PosixPath('/usr/bin/sudo'), PosixPath('/usr/bin/cpufreq-set'), '-g', 'powersave') @@ -72,7 +73,8 @@ def test_throttle_on(): def test_throttle_off(): - with mock.patch('wrolpi.admin.subprocess') as mock_subprocess: + with mock.patch('wrolpi.admin.subprocess') as mock_subprocess, \ + mock.patch('wrolpi.admin.CPUFREQ_SET', PosixPath('/usr/bin/cpufreq-set')): assert admin.throttle_cpu_off() mock_subprocess.check_call.assert_called_once_with( (PosixPath('/usr/bin/sudo'), PosixPath('/usr/bin/cpufreq-set'), '-g', 'ondemand') diff --git a/wrolpi/test/test_cmd.py b/wrolpi/test/test_cmd.py new file mode 100644 index 00000000..43613ab0 --- /dev/null +++ b/wrolpi/test/test_cmd.py @@ -0,0 +1,22 @@ +import os +from pathlib import Path + +from wrolpi import cmd + + +def test_which(test_directory): + # Default location in Debian Linux. + assert cmd.which('ls') == Path('/usr/bin/ls') + # This executable does not exist. + assert cmd.which('asdf') is None + # The alternative path does not exist either. + assert cmd.which('asdf', 'asdf') is None + # Directories are ignored. + assert cmd.which('asdf', '/tmp') is None + + # Create an executable. + foo = test_directory / 'foo' + foo.touch() + assert cmd.which('asdf', str(foo)) is None + os.chmod(foo, 0o777) + assert cmd.which('asdf', str(foo)) == foo diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 35e0095b..5f02d320 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -61,8 +61,9 @@ class TestRootAPI(TestAPI): assert json.loads(response.body) == {'valid': False, 'regex': '.*(missing parenthesis.*'} def test_get_settings(self): - request, response = api_app.test_client.get('/api/settings') - self.assertOK(response) + with mock.patch('wrolpi.admin.CPUFREQ_INFO', '/usr/bin/cpufreq-info'): + request, response = api_app.test_client.get('/api/settings') + self.assertOK(response) @wrap_test_db def test_get_downloads(self):