API will start without DB.

This commit is contained in:
Roland 2023-01-31 17:11:31 -07:00
parent 76291d2258
commit 163b14c294
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
10 changed files with 111 additions and 48 deletions

View file

@ -11,6 +11,7 @@ import {ProgressPlaceholder} from "./components/Placeholder";
import {Divider, Icon, Message} from "semantic-ui-react";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {refreshFiles} from "./api";
import _ from "lodash";
function FlagsMessages({flags}) {
if (!flags) {
@ -18,7 +19,8 @@ function FlagsMessages({flags}) {
}
let refreshing;
let refreshComplete;
let refreshRequired;
let dbDown;
// Do not tell the maintainer to refresh the files if they are already refreshing.
if (flags.indexOf('refreshing') >= 0) {
@ -32,7 +34,7 @@ function FlagsMessages({flags}) {
</Message>;
} else if (flags.indexOf('refresh_complete') === -1) {
// `refresh_complete` flag is not set. Tell the maintainer to refresh the files.
refreshComplete = <Message icon warning onClick={refreshFiles}>
refreshRequired = <Message icon warning onClick={refreshFiles}>
<Icon name='hand point right'/>
<Message.Content>
<Message.Header>Refresh required</Message.Header>
@ -41,9 +43,19 @@ function FlagsMessages({flags}) {
</Message>;
}
if (flags.indexOf('db_up') === -1) {
dbDown = <Message icon error>
<Icon name='exclamation'/>
<Message.Content>
<Message.Header>Database is down</Message.Header>
API is unable to connect to the database. Check the server logs.
</Message.Content>
</Message>
}
return <>
{refreshing}
{refreshComplete}
{dbDown || refreshRequired}
</>
}
@ -106,8 +118,8 @@ function DashboardStatus() {
}
const {downloads} = status;
if (downloads) {
pending_downloads = downloads['disabled'] ? 'x' : downloads['pending'];
if (!_.isEmpty(downloads)) {
pending_downloads = downloads && downloads['disabled'] ? 'x' : downloads['pending'];
}
let bandwidths = <ProgressPlaceholder/>;

View file

@ -48,18 +48,7 @@ async function apiCall(url, method, body, ms = 20000) {
return response;
} catch (e) {
// Timeout, or some other exception.
if (e.message === 'promise timeout') {
toast({
type: 'error',
title: 'Server failed to respond!',
description: 'Timeout while waiting for server response. See server logs.',
time: 5000,
});
} else {
toast({
type: 'error', title: 'Error!', description: 'See client logs', time: 5000,
});
}
console.error(e);
throw e;
}
}

41
main.py
View file

@ -1,5 +1,6 @@
#! /usr/bin/env python3
import argparse
import asyncio
import logging
import sys
@ -7,7 +8,7 @@ from sanic import Sanic
from sanic.signals import Event
from wrolpi import flags
from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, admin
from wrolpi import root_api, admin
from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, \
wrol_mode_enabled, cancel_refresh_tasks
from wrolpi.downloader import download_manager, import_downloads_config
@ -109,6 +110,10 @@ def main():
print('Media directory is correct.')
return 0
if not args.sub_commands:
parser.print_help()
return 1
logger.warning(f'Starting with: {sys.argv}')
set_log_level(args)
logger.debug(get_version_string())
@ -133,16 +138,17 @@ def main():
# Import the API in every module. Each API should attach itself to `root_api`.
import_modules()
# Run the startup functions
for func in BEFORE_STARTUP_FUNCTIONS:
try:
logger.debug(f'Calling {func} before startup.')
func()
except Exception as e:
logger.warning(f'Startup {func} failed!', exc_info=e)
# # Run the startup functions
# for func in BEFORE_STARTUP_FUNCTIONS:
# try:
# logger.debug(f'Calling {func} before startup.')
# func()
# except Exception as e:
# logger.warning(f'Startup {func} failed!', exc_info=e)
# Run the API.
return root_api.main(args)
if args.sub_commands == 'api':
return root_api.main(args)
def set_log_level(args):
@ -166,11 +172,24 @@ def set_log_level(args):
@api_app.before_server_start
@limit_concurrent(1)
async def startup(app: Sanic):
# Check database status first. Many function will reference flags.db_up.
flags.check_db_is_up()
flags.init_flags()
await import_downloads_config()
@api_app.after_server_start
@limit_concurrent(1)
async def periodic_check_db_is_up(app: Sanic):
while True:
flags.check_db_is_up()
flags.init_flags()
await asyncio.sleep(10)
@api_app.after_server_start
async def periodic_downloads(app: Sanic):
"""
@ -178,6 +197,10 @@ async def periodic_downloads(app: Sanic):
Limited to only one process.
"""
if not flags.db_up.is_set():
logger.warning(f'Refusing to download when DB is not up.')
return
if not flags.refresh_complete.is_set():
logger.warning('Refusing to download without refresh')
download_manager.disable()

View file

@ -509,7 +509,7 @@ def check_media_directory():
media_directory = get_media_directory()
if not media_directory.is_dir():
logger_.error(f'Media directory does not exist: {media_directory}')
result = False
return False
permissions = media_directory.stat().st_mode
if permissions != MEDIA_DIRECTORY_PERMISSIONS:

View file

@ -43,9 +43,7 @@ def get_test_db_engine():
def test_db() -> Tuple[Engine, Session]:
"""
Create a unique SQLAlchemy engine/session for a test.
"""
"""Create a unique SQLAlchemy engine/session for a test."""
test_engine = get_test_db_engine()
if test_engine.engine.url.database == 'wrolpi':
raise ValueError('Refusing the test on wrolpi database!')
@ -58,9 +56,7 @@ def test_db() -> Tuple[Engine, Session]:
@pytest.fixture()
def test_session() -> Session:
"""
Pytest Fixture to get a test database session.
"""
"""Pytest Fixture to get a test database session."""
test_engine, session = test_db()
def fake_get_db_session():

View file

@ -41,7 +41,7 @@ postgres_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{d
# This engine is used for all normal tasks (except testing).
db_args = get_db_args()
connect_args = dict(application_name='wrolpi_api', connect_timeout=600)
connect_args = dict(application_name='wrolpi_api', connect_timeout=1)
uri = 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**db_args)
engine = create_engine(uri, poolclass=NullPool, connect_args=connect_args)
session_maker = sessionmaker(bind=engine)

View file

@ -23,6 +23,7 @@ from sqlalchemy import Column, Integer, String, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \
background_task, limit_concurrent
from wrolpi.dates import TZDateTime, now, Seconds
@ -807,8 +808,10 @@ class DownloadManager:
self.disabled.set()
for downloader in self.instances:
downloader.kill()
for download in self.get_pending_downloads():
download.defer()
if flags.db_up.is_set():
# Only defer downloads if the DB is up.
for download in self.get_pending_downloads():
download.defer()
self.cancel_workers()
def stop(self):
@ -1092,7 +1095,8 @@ async def import_downloads_config(session: Session):
"""Upsert all Downloads in the Download Manager Config into the DB.
The config is the source of truth."""
if not DOWNLOAD_MANAGER_CONFIG.downloads:
if not PYTEST and not flags.db_up.is_set():
logger.warning(f'Refusing to import downloads config when DB is not up.')
return
try:

View file

@ -1,4 +1,5 @@
import multiprocessing
import subprocess
import threading
from typing import List
@ -95,6 +96,8 @@ class Flag:
logger.critical(f'Unable to save flag! {repr(self)}', exc_info=e)
# Set by `check_db_is_up`.
db_up = Flag('db_up')
# The global refresh is running.
refreshing = Flag('refreshing')
# The global refresh has been performed. This is False on a fresh instance of WROLPi.
@ -112,6 +115,8 @@ yt_dlp_installed = Flag('yt_dlp_installed')
def get_flags() -> List[str]:
"""Return a list of all Flags which are set."""
flags = []
if db_up.is_set():
flags.append('db_up')
if refreshing.is_set():
flags.append('refreshing')
if refresh_complete.is_set():
@ -142,8 +147,44 @@ class WROLPiFlag(Base):
return f'<WROLPiFlag refresh_complete={self.refresh_complete}>'
def check_db_is_up():
"""Attempts to connect to the database, sets flags.db_up if successful."""
from wrolpi.db import get_db_curs, get_db_args
db_args = get_db_args()
db_host = db_args['host']
if db_host != '127.0.0.1':
# Check if database host is up before attempting connection. This will fail faster than SQLAlchemy timeout.
try:
cmd = ['ping', '-w', '1', '-c', '1', db_host]
subprocess.check_call(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
except Exception as e:
logger.debug(f'Unable to resolve database host {db_host}')
db_up.clear()
return
try:
with get_db_curs() as curs:
curs.execute('SELECT * FROM alembic_version')
# If we get here, the database is up!
db_up.set()
except Exception as e:
logger.debug(f'DB is not up', exc_info=e)
db_up.clear()
FLAGS_INITIALIZED = multiprocessing.Event()
def init_flags():
"""Set flags to match their DB values."""
if FLAGS_INITIALIZED.is_set():
return
if not db_up.is_set():
logger.error(f'Refusing to initialize flags when DB is not up.')
return
from wrolpi.db import get_db_session
with get_db_session() as session:
flags: WROLPiFlag = session.query(WROLPiFlag).one_or_none()

View file

@ -14,7 +14,7 @@ from sanic.response import HTTPResponse
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi import admin, status
from wrolpi import admin, status, flags
from wrolpi.admin import HotspotStatus
from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics
@ -23,7 +23,6 @@ from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, API_ERRORS, APIError, ValidationError, HotspotError
from wrolpi.events import get_events
from wrolpi.files.lib import get_file_statistics
from wrolpi.flags import get_flags
from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse, \
EventsRequest
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
@ -331,13 +330,12 @@ async def throttle_off(_: Request):
@openapi.description('Get the status of CPU/load/etc.')
async def get_status(_: Request):
s = await status.get_status()
try:
downloads = download_manager.get_summary()
except Exception as e:
logger.error('Unable to get download status', exc_info=e)
downloads = dict()
flags = get_flags()
downloads = dict()
if flags.db_up.is_set():
try:
downloads = download_manager.get_summary()
except Exception as e:
logger.debug('Unable to get download status', exc_info=e)
ret = dict(
bandwidth=s.bandwidth,
@ -346,7 +344,7 @@ async def get_status(_: Request):
dockerized=DOCKERIZED,
downloads=downloads,
drives=s.drives,
flags=flags,
flags=flags.get_flags(),
hotspot_status=admin.hotspot_status().name,
load=s.load,
memory_stats=s.memory_stats,

View file

@ -409,7 +409,7 @@ def test_crud_download(test_client, test_session, test_download_manager):
@pytest.mark.asyncio
async def test_downloads_config(test_client, test_session, test_download_manager, test_download_manager_config):
async def test_downloads_config(test_session, test_client, test_download_manager, test_download_manager_config):
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
def assert_downloads(expected_):