diff --git a/main.py b/main.py index b083f8d6..91748324 100755 --- a/main.py +++ b/main.py @@ -13,7 +13,7 @@ from wrolpi import flags from wrolpi import root_api # noqa from wrolpi.api_utils import api_app, perpetual_signal, per_worker_task from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \ - cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled, \ + cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, have_internet_connection, wrol_mode_enabled, \ create_empty_config_files, import_file_configs, TRACE_LEVEL from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts from wrolpi.dates import Seconds @@ -22,7 +22,7 @@ from wrolpi.downloader import download_manager, get_download_manager_config from wrolpi.errors import WROLModeEnabled from wrolpi.files.worker import file_worker from wrolpi.perpetual import register_perpetual_process, perpetual_process_health_check -from wrolpi.vars import PROJECT_DIR, DOCKERIZED, INTERNET_SERVER +from wrolpi.vars import PROJECT_DIR, DOCKERIZED from wrolpi.version import get_version_string logger = logger.getChild('wrolpi-main') @@ -339,7 +339,8 @@ async def perpetual_check_db_is_up_worker(): @perpetual_signal(sleep=10) async def perpetual_have_internet_worker(): try: - if can_connect_to_server(INTERNET_SERVER): + # In a thread: the check blocks, and would stall every other perpetual loop. + if await asyncio.to_thread(have_internet_connection): flags.have_internet.set() # Check hourly once we have internet. await asyncio.sleep(float(Seconds.hour)) diff --git a/wrolpi/common.py b/wrolpi/common.py index 897dd9d4..747599eb 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -268,6 +268,7 @@ __all__ = [ 'apply_refresh_cleanup', 'background_task', 'can_connect_to_server', + 'have_internet_connection', 'cancel_background_tasks', 'cancel_refresh_tasks', 'cancelable_wrapper', @@ -1417,6 +1418,8 @@ class WROLPiConfigValidator: hotspot_password: str = None hotspot_ssid: str = None ignore_outdated_zims: bool = None + internet_check_hostname: str = None + internet_check_ip: str = None log_level: str = None map_default_location: dict = None map_destination: str = None @@ -1697,6 +1700,9 @@ class WROLPiConfig(ConfigFile): check_for_upgrades=True, ignore_outdated_zims=False, ignored_directories=['config', 'tags', REPOS_DIRECTORY], + # The Internet is up when either answers (see `have_internet_connection`). + internet_check_hostname='one.one.one.one', + internet_check_ip='1.1.1.1', log_level='info', map_default_location=None, map_destination='map', @@ -1790,6 +1796,22 @@ class WROLPiConfig(ConfigFile): def require_cookies_unlocked(self, value: bool): self.update({'require_cookies_unlocked': value}) + @property + def internet_check_hostname(self) -> Optional[str]: + return self._config.get('internet_check_hostname') + + @internet_check_hostname.setter + def internet_check_hostname(self, value: Optional[str]): + self.update({'internet_check_hostname': value}) + + @property + def internet_check_ip(self) -> Optional[str]: + return self._config.get('internet_check_ip') + + @internet_check_ip.setter + def internet_check_ip(self, value: Optional[str]): + self.update({'internet_check_ip': value}) + @property def require_media_mounted(self) -> bool: return self._config.get('require_media_mounted', True) @@ -3369,14 +3391,18 @@ def get_title_from_html(html: str, url: str = None) -> str: logger_.debug(f'Unable to extract title {url}') +# Seconds to wait for a server to answer. A firewall that drops the connection would otherwise take minutes to fail. +INTERNET_CHECK_TIMEOUT = 5 + + @wrol_mode_check -def can_connect_to_server(hostname: str, port: int = 80) -> bool: +def can_connect_to_server(hostname: str, port: int = 80, timeout: float = INTERNET_CHECK_TIMEOUT) -> bool: """Return True only if `hostname` responds.""" # Thanks https://stackoverflow.com/questions/20913411/test-if-an-internet-connection-is-present-in-python try: # connect to the host -- tells us if the host is actually # reachable - conn = socket.create_connection((hostname, port)) + conn = socket.create_connection((hostname, port), timeout=timeout) conn.close() return True except Exception as e: @@ -3384,6 +3410,17 @@ def can_connect_to_server(hostname: str, port: int = 80) -> bool: return False +def have_internet_connection() -> bool: + """Whether the Internet is up: wrolpi.yaml's `internet_check_ip` or its `internet_check_hostname` answers. Blocks; + call it from a thread. + + The IP address is tried first: the hostname needs DNS, and a name lookup is not bounded by the connection's + timeout.""" + config = get_wrolpi_config() + hosts = [i for i in (config.internet_check_ip, config.internet_check_hostname) if i] + return any(can_connect_to_server(i) for i in hosts) + + def is_valid_hex_color(hex_color: str) -> bool: # Regular expression to match valid HTML hex color codes hex_color_regex = re.compile(r'^#([A-F0-9]{6}|[A-F0-9]{3})$', re.IGNORECASE) diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 09fd7679..b38e98e5 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -856,6 +856,56 @@ def test_can_connect_to_server(simple_file_server): assert common.can_connect_to_server(host, port) +def test_can_connect_to_server_times_out(monkeypatch): + """A server that never answers (a firewall that drops the connection) gives up after the timeout, instead of the + system's default of minutes.""" + timeouts = [] + + def create_connection(address, timeout=None): + timeouts.append(timeout) + raise socket.timeout() + + monkeypatch.setattr(common.socket, 'create_connection', create_connection) + + assert common.can_connect_to_server('example.com', timeout=2) is False + assert common.can_connect_to_server('example.com') is False + assert timeouts == [2, common.INTERNET_CHECK_TIMEOUT] + + +@pytest.mark.asyncio +async def test_have_internet_connection(async_client, test_wrolpi_config, monkeypatch): + """The Internet is up when wrolpi.yaml's IP address or its hostname answers. The IP address is tried first: the + hostname needs DNS, which the connection's timeout does not bound.""" + answering = set() + checked = [] + + def can_connect_to_server(hostname, port=80, timeout=None): + checked.append(hostname) + return hostname in answering + + monkeypatch.setattr(common, 'can_connect_to_server', can_connect_to_server) + config = get_wrolpi_config() + assert (config.internet_check_hostname, config.internet_check_ip) == ('one.one.one.one', '1.1.1.1') + + config.internet_check_hostname = 'internet.example.com' + config.internet_check_ip = '192.0.2.1' + assert common.have_internet_connection() is False + assert checked == ['192.0.2.1', 'internet.example.com'] + + # A working IP address answers without a name lookup. + answering.add('192.0.2.1') + checked.clear() + assert common.have_internet_connection() is True + assert checked == ['192.0.2.1'] + + # Only the hostname is checked when there is no IP address. + config.internet_check_ip = None + checked.clear() + answering.add('internet.example.com') + assert common.have_internet_connection() is True + assert checked == ['internet.example.com'] + + @pytest.mark.asyncio async def test_log_level(async_client, test_wrolpi_config): """User can change API log level.""" diff --git a/wrolpi/vars.py b/wrolpi/vars.py index c2ae7f2a..320d3bac 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -54,9 +54,6 @@ DOCKERIZED = truthy_arg(os.environ.get('DOCKER', '')) # tests are running PYTEST = 'pytest' in sys.modules -# Used to test if the Internet is up. -INTERNET_SERVER = os.environ.get('INTERNET_SERVER', 'one.one.one.one') - # Running on a GitHub Actions runner, which lacks things some tests need. IS_CI = truthy_arg(os.environ.get('GITHUB_ACTIONS', ''))