From 163b14c2941852a9143f25838d3f6aaa76eb421a Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 31 Jan 2023 17:11:31 -0700 Subject: [PATCH] API will start without DB. --- app/src/Dashboard.js | 22 +++++++++++++----- app/src/api.js | 13 +---------- main.py | 41 ++++++++++++++++++++++++++-------- wrolpi/common.py | 2 +- wrolpi/conftest.py | 8 ++----- wrolpi/db.py | 2 +- wrolpi/downloader.py | 10 ++++++--- wrolpi/flags.py | 41 ++++++++++++++++++++++++++++++++++ wrolpi/root_api.py | 18 +++++++-------- wrolpi/test/test_downloader.py | 2 +- 10 files changed, 111 insertions(+), 48 deletions(-) diff --git a/app/src/Dashboard.js b/app/src/Dashboard.js index 3819a797..a15e5746 100644 --- a/app/src/Dashboard.js +++ b/app/src/Dashboard.js @@ -11,6 +11,7 @@ import {ProgressPlaceholder} from "./components/Placeholder"; import {Divider, Icon, Message} from "semantic-ui-react"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import {refreshFiles} from "./api"; +import _ from "lodash"; function FlagsMessages({flags}) { if (!flags) { @@ -18,7 +19,8 @@ function FlagsMessages({flags}) { } let refreshing; - let refreshComplete; + let refreshRequired; + let dbDown; // Do not tell the maintainer to refresh the files if they are already refreshing. if (flags.indexOf('refreshing') >= 0) { @@ -32,7 +34,7 @@ function FlagsMessages({flags}) { ; } else if (flags.indexOf('refresh_complete') === -1) { // `refresh_complete` flag is not set. Tell the maintainer to refresh the files. - refreshComplete = + refreshRequired = Refresh required @@ -41,9 +43,19 @@ function FlagsMessages({flags}) { ; } + if (flags.indexOf('db_up') === -1) { + dbDown = + + + Database is down + API is unable to connect to the database. Check the server logs. + + + } + return <> {refreshing} - {refreshComplete} + {dbDown || refreshRequired} } @@ -106,8 +118,8 @@ function DashboardStatus() { } const {downloads} = status; - if (downloads) { - pending_downloads = downloads['disabled'] ? 'x' : downloads['pending']; + if (!_.isEmpty(downloads)) { + pending_downloads = downloads && downloads['disabled'] ? 'x' : downloads['pending']; } let bandwidths = ; diff --git a/app/src/api.js b/app/src/api.js index 4fc977e8..5ed35171 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -48,18 +48,7 @@ async function apiCall(url, method, body, ms = 20000) { return response; } catch (e) { // Timeout, or some other exception. - if (e.message === 'promise timeout') { - toast({ - type: 'error', - title: 'Server failed to respond!', - description: 'Timeout while waiting for server response. See server logs.', - time: 5000, - }); - } else { - toast({ - type: 'error', title: 'Error!', description: 'See client logs', time: 5000, - }); - } + console.error(e); throw e; } } diff --git a/main.py b/main.py index 5f669058..6370799f 100755 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ #! /usr/bin/env python3 import argparse +import asyncio import logging import sys @@ -7,7 +8,7 @@ from sanic import Sanic from sanic.signals import Event from wrolpi import flags -from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, admin +from wrolpi import root_api, admin from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, \ wrol_mode_enabled, cancel_refresh_tasks from wrolpi.downloader import download_manager, import_downloads_config @@ -109,6 +110,10 @@ def main(): print('Media directory is correct.') return 0 + if not args.sub_commands: + parser.print_help() + return 1 + logger.warning(f'Starting with: {sys.argv}') set_log_level(args) logger.debug(get_version_string()) @@ -133,16 +138,17 @@ def main(): # Import the API in every module. Each API should attach itself to `root_api`. import_modules() - # Run the startup functions - for func in BEFORE_STARTUP_FUNCTIONS: - try: - logger.debug(f'Calling {func} before startup.') - func() - except Exception as e: - logger.warning(f'Startup {func} failed!', exc_info=e) + # # Run the startup functions + # for func in BEFORE_STARTUP_FUNCTIONS: + # try: + # logger.debug(f'Calling {func} before startup.') + # func() + # except Exception as e: + # logger.warning(f'Startup {func} failed!', exc_info=e) # Run the API. - return root_api.main(args) + if args.sub_commands == 'api': + return root_api.main(args) def set_log_level(args): @@ -166,11 +172,24 @@ def set_log_level(args): @api_app.before_server_start +@limit_concurrent(1) async def startup(app: Sanic): + # Check database status first. Many function will reference flags.db_up. + flags.check_db_is_up() + flags.init_flags() await import_downloads_config() +@api_app.after_server_start +@limit_concurrent(1) +async def periodic_check_db_is_up(app: Sanic): + while True: + flags.check_db_is_up() + flags.init_flags() + await asyncio.sleep(10) + + @api_app.after_server_start async def periodic_downloads(app: Sanic): """ @@ -178,6 +197,10 @@ async def periodic_downloads(app: Sanic): Limited to only one process. """ + if not flags.db_up.is_set(): + logger.warning(f'Refusing to download when DB is not up.') + return + if not flags.refresh_complete.is_set(): logger.warning('Refusing to download without refresh') download_manager.disable() diff --git a/wrolpi/common.py b/wrolpi/common.py index e4c9e6c2..b2b76ae5 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -509,7 +509,7 @@ def check_media_directory(): media_directory = get_media_directory() if not media_directory.is_dir(): logger_.error(f'Media directory does not exist: {media_directory}') - result = False + return False permissions = media_directory.stat().st_mode if permissions != MEDIA_DIRECTORY_PERMISSIONS: diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index b54b4931..a3004166 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -43,9 +43,7 @@ def get_test_db_engine(): def test_db() -> Tuple[Engine, Session]: - """ - Create a unique SQLAlchemy engine/session for a test. - """ + """Create a unique SQLAlchemy engine/session for a test.""" test_engine = get_test_db_engine() if test_engine.engine.url.database == 'wrolpi': raise ValueError('Refusing the test on wrolpi database!') @@ -58,9 +56,7 @@ def test_db() -> Tuple[Engine, Session]: @pytest.fixture() def test_session() -> Session: - """ - Pytest Fixture to get a test database session. - """ + """Pytest Fixture to get a test database session.""" test_engine, session = test_db() def fake_get_db_session(): diff --git a/wrolpi/db.py b/wrolpi/db.py index 35d672b9..6ac44dae 100644 --- a/wrolpi/db.py +++ b/wrolpi/db.py @@ -41,7 +41,7 @@ postgres_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{d # This engine is used for all normal tasks (except testing). db_args = get_db_args() -connect_args = dict(application_name='wrolpi_api', connect_timeout=600) +connect_args = dict(application_name='wrolpi_api', connect_timeout=1) uri = 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**db_args) engine = create_engine(uri, poolclass=NullPool, connect_args=connect_args) session_maker = sessionmaker(bind=engine) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 10f7ba50..0db450e3 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -23,6 +23,7 @@ from sqlalchemy import Column, Integer, String, Text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Session +from wrolpi import flags from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \ background_task, limit_concurrent from wrolpi.dates import TZDateTime, now, Seconds @@ -807,8 +808,10 @@ class DownloadManager: self.disabled.set() for downloader in self.instances: downloader.kill() - for download in self.get_pending_downloads(): - download.defer() + if flags.db_up.is_set(): + # Only defer downloads if the DB is up. + for download in self.get_pending_downloads(): + download.defer() self.cancel_workers() def stop(self): @@ -1092,7 +1095,8 @@ async def import_downloads_config(session: Session): """Upsert all Downloads in the Download Manager Config into the DB. The config is the source of truth.""" - if not DOWNLOAD_MANAGER_CONFIG.downloads: + if not PYTEST and not flags.db_up.is_set(): + logger.warning(f'Refusing to import downloads config when DB is not up.') return try: diff --git a/wrolpi/flags.py b/wrolpi/flags.py index 6289f1a5..9a904b3f 100644 --- a/wrolpi/flags.py +++ b/wrolpi/flags.py @@ -1,4 +1,5 @@ import multiprocessing +import subprocess import threading from typing import List @@ -95,6 +96,8 @@ class Flag: logger.critical(f'Unable to save flag! {repr(self)}', exc_info=e) +# Set by `check_db_is_up`. +db_up = Flag('db_up') # The global refresh is running. refreshing = Flag('refreshing') # The global refresh has been performed. This is False on a fresh instance of WROLPi. @@ -112,6 +115,8 @@ yt_dlp_installed = Flag('yt_dlp_installed') def get_flags() -> List[str]: """Return a list of all Flags which are set.""" flags = [] + if db_up.is_set(): + flags.append('db_up') if refreshing.is_set(): flags.append('refreshing') if refresh_complete.is_set(): @@ -142,8 +147,44 @@ class WROLPiFlag(Base): return f'' +def check_db_is_up(): + """Attempts to connect to the database, sets flags.db_up if successful.""" + from wrolpi.db import get_db_curs, get_db_args + + db_args = get_db_args() + db_host = db_args['host'] + if db_host != '127.0.0.1': + # Check if database host is up before attempting connection. This will fail faster than SQLAlchemy timeout. + try: + cmd = ['ping', '-w', '1', '-c', '1', db_host] + subprocess.check_call(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + except Exception as e: + logger.debug(f'Unable to resolve database host {db_host}') + db_up.clear() + return + + try: + with get_db_curs() as curs: + curs.execute('SELECT * FROM alembic_version') + # If we get here, the database is up! + db_up.set() + except Exception as e: + logger.debug(f'DB is not up', exc_info=e) + db_up.clear() + + +FLAGS_INITIALIZED = multiprocessing.Event() + + def init_flags(): """Set flags to match their DB values.""" + if FLAGS_INITIALIZED.is_set(): + return + + if not db_up.is_set(): + logger.error(f'Refusing to initialize flags when DB is not up.') + return + from wrolpi.db import get_db_session with get_db_session() as session: flags: WROLPiFlag = session.query(WROLPiFlag).one_or_none() diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 95db4512..e5dc87fc 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -14,7 +14,7 @@ from sanic.response import HTTPResponse from sanic_ext import validate from sanic_ext.extensions.openapi import openapi -from wrolpi import admin, status +from wrolpi import admin, status, flags from wrolpi.admin import HotspotStatus from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \ wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics @@ -23,7 +23,6 @@ from wrolpi.downloader import download_manager from wrolpi.errors import WROLModeEnabled, API_ERRORS, APIError, ValidationError, HotspotError from wrolpi.events import get_events from wrolpi.files.lib import get_file_statistics -from wrolpi.flags import get_flags from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse, \ EventsRequest from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \ @@ -331,13 +330,12 @@ async def throttle_off(_: Request): @openapi.description('Get the status of CPU/load/etc.') async def get_status(_: Request): s = await status.get_status() - try: - downloads = download_manager.get_summary() - except Exception as e: - logger.error('Unable to get download status', exc_info=e) - downloads = dict() - - flags = get_flags() + downloads = dict() + if flags.db_up.is_set(): + try: + downloads = download_manager.get_summary() + except Exception as e: + logger.debug('Unable to get download status', exc_info=e) ret = dict( bandwidth=s.bandwidth, @@ -346,7 +344,7 @@ async def get_status(_: Request): dockerized=DOCKERIZED, downloads=downloads, drives=s.drives, - flags=flags, + flags=flags.get_flags(), hotspot_status=admin.hotspot_status().name, load=s.load, memory_stats=s.memory_stats, diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 0bd9bec5..d23d8feb 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -409,7 +409,7 @@ def test_crud_download(test_client, test_session, test_download_manager): @pytest.mark.asyncio -async def test_downloads_config(test_client, test_session, test_download_manager, test_download_manager_config): +async def test_downloads_config(test_session, test_client, test_download_manager, test_download_manager_config): from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG def assert_downloads(expected_):