wrolpi/main.py
Roland 758ff8e810
Creating background process that fetches comments for Videos that do not yet have comments.
Displaying star for Uploader's comments, heart for favorited comments.

Adding wrolpi help files desktop shortcut to rpi image.

Using zim file names as FileGroup.title.

Removing accidental output commit.
2024-03-01 17:15:56 -07:00

288 lines
8.8 KiB
Python
Executable file

#! /usr/bin/env python3
import argparse
import asyncio
import logging
import sys
from sanic import Sanic
from sanic.signals import Event
from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS
from wrolpi import root_api, admin
from wrolpi.common import logger, get_config, check_media_directory, limit_concurrent, \
wrol_mode_enabled, cancel_refresh_tasks, set_log_level, background_task, cancel_background_tasks
from wrolpi.dates import Seconds
from wrolpi.downloader import download_manager, import_downloads_config
from wrolpi.root_api import api_app
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
from wrolpi.version import get_version_string
logger = logger.getChild('wrolpi-main')
def db_main(args):
"""
Handle database migrations. Currently this uses Alembic, supported commands are "upgrade" and "downgrade".
"""
from alembic.config import Config
from alembic import command
from wrolpi.db import uri
config = Config(PROJECT_DIR / 'alembic.ini')
# Overwrite the Alembic config, the is usually necessary when running in a docker container.
config.set_main_option('sqlalchemy.url', uri)
logger.warning(f'DB URI: {uri}')
if args.command == 'upgrade':
command.upgrade(config, 'head')
elif args.command == 'downgrade':
command.downgrade(config, '-1')
else:
print(f'Unknown DB command: {args.command}')
return 2
return 0
INTERACTIVE_BANNER = '''
This is the interactive WROLPi shell. Use this to interact with the WROLPi API library.
Example (get the duration of every video file):
from modules.videos.models import Video
from modules.videos.common import get_video_duration
videos = session.query(Video).filter(Video.video_path != None).all()
videos = list(videos)
for video in videos:
get_video_duration(video.video_path.path)
Check local variables:
locals().keys()
'''
def launch_interactive_shell():
"""Launches an interactive shell with a DB session."""
import code
from wrolpi.db import get_db_session
with get_db_session() as session:
code.interact(banner=INTERACTIVE_BANNER, local=locals())
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='count')
parser.add_argument('--version', action='store_true', default=False)
parser.add_argument('-c', '--check-media', action='store_true', default=False,
help='Check that the media directory is mounted and has the correct permissions.'
)
parser.add_argument('-i', '--interactive', action='store_true', default=False,
help='Enter an interactive shell with some WROLPi tools')
sub_commands = parser.add_subparsers(title='sub-commands', dest='sub_commands')
# Add the API parser, this will allow the user to specify host/port etc.
api_parser = sub_commands.add_parser('api')
root_api.init_parser(api_parser)
# DB Parser for running Alembic migrations
db_parser = sub_commands.add_parser('db')
db_parser.add_argument('command', help='Supported commands: upgrade, downgrade')
args = parser.parse_args()
if args.interactive:
launch_interactive_shell()
return 0
if args.version:
# Print out the relevant version information, then exit.
print(get_version_string())
return 0
if args.check_media:
# Run the media directory check. Exit with informative return code.
result = check_media_directory()
if result is False:
return 1
print('Media directory is correct.')
return 0
if not args.sub_commands:
parser.print_help()
return 1
logger.warning(f'Starting with: {sys.argv}')
from wrolpi.common import LOG_LEVEL
with LOG_LEVEL.get_lock():
if args.verbose == 1:
LOG_LEVEL.value = logging.INFO
set_log_level(logging.INFO)
elif args.verbose and args.verbose == 2:
LOG_LEVEL.value = logging.DEBUG
set_log_level(logging.DEBUG)
elif args.verbose and args.verbose >= 3:
# Log everything. Add SQLAlchemy debug logging.
LOG_LEVEL.value = logging.NOTSET
set_log_level(logging.NOTSET)
logger.info(get_version_string())
if DOCKERIZED:
logger.info('Running in Docker')
# Run DB migrations before anything else.
if args.sub_commands == 'db':
return db_main(args)
config = get_config()
# Hotspot/throttle are not supported in Docker containers.
if not DOCKERIZED and config.hotspot_on_startup:
try:
admin.enable_hotspot()
except Exception as e:
logger.error('Failed to enable hotspot', exc_info=e)
if not DOCKERIZED and config.throttle_on_startup:
try:
admin.throttle_cpu_on()
except Exception as e:
logger.error('Failed to throttle CPU', exc_info=e)
check_media_directory()
# Import modules before calling BEFORE_STARTUP_FUNCTIONS.
import modules # noqa
# Run the startup functions
for func in BEFORE_STARTUP_FUNCTIONS:
try:
logger.debug(f'Calling {func} before startup.')
func()
except Exception as e:
logger.warning(f'Startup {func} failed!', exc_info=e)
# Run the API.
if args.sub_commands == 'api':
return root_api.main(args)
@api_app.before_server_start
@limit_concurrent(1)
async def startup(app: Sanic):
from wrolpi.common import LOG_LEVEL
# Check database status first. Many functions will reference flags.db_up.
flags.check_db_is_up()
flags.init_flags()
await import_downloads_config()
async def periodic_check_db_is_up():
while True:
flags.check_db_is_up()
flags.init_flags()
await asyncio.sleep(10)
background_task(periodic_check_db_is_up())
async def periodic_check_log_level():
while True:
log_level = LOG_LEVEL.value
if log_level != logger.getEffectiveLevel():
set_log_level(log_level)
await asyncio.sleep(1)
background_task(periodic_check_log_level())
from wrolpi import status
background_task(status.bandwidth_worker())
from modules.zim.lib import flag_outdated_zim_files
flag_outdated_zim_files()
from modules.videos.video.lib import get_missing_videos_comments
async def periodic_start_video_missing_comments_download():
async with flags.refresh_complete.wait_for():
# We can't search for Videos missing comments until the refresh has completed.
pass
while True:
# Fetch comments for videos every hour.
if download_manager.disabled.is_set() or download_manager.stopped.is_set():
await asyncio.sleep(10)
else:
await get_missing_videos_comments()
await asyncio.sleep(int(Seconds.hour))
background_task(periodic_start_video_missing_comments_download())
@api_app.after_server_start
async def periodic_downloads(app: Sanic):
"""
Starts the perpetual downloader on download manager.
Limited to only one process.
"""
async with flags.db_up.wait_for():
pass
if not flags.refresh_complete.is_set():
logger.warning('Refusing to download without refresh')
download_manager.disable()
return
# Set all downloads to new.
download_manager.reset_downloads()
if wrol_mode_enabled():
logger.warning('Not starting download manager because WROL Mode is enabled.')
download_manager.disable()
return
config = get_config()
if config.download_on_startup is False:
logger.warning('Not starting download manager because Downloads are disabled on startup.')
download_manager.disable()
return
async with flags.db_up.wait_for():
download_manager.enable()
app.add_task(download_manager.perpetual_download())
@api_app.after_server_start
async def start_workers(app: Sanic):
"""All Sanic processes have their own Download workers."""
if wrol_mode_enabled():
logger.warning(f'Not starting download workers because WROL Mode is enabled.')
download_manager.stop()
return
async with flags.db_up.wait_for():
download_manager.start_workers()
@api_app.before_server_start
@limit_concurrent(1)
async def main_import_tags_config(app: Sanic):
from wrolpi import tags
async with flags.db_up.wait_for():
tags.import_tags_config()
@root_api.api_app.signal(Event.SERVER_SHUTDOWN_BEFORE)
@limit_concurrent(1)
async def handle_server_shutdown(*args, **kwargs):
"""Stop downloads when server is shutting down."""
if not PYTEST:
download_manager.stop()
await cancel_refresh_tasks()
await cancel_background_tasks()
if __name__ == '__main__':
sys.exit(main())