mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
319 lines
11 KiB
Python
319 lines
11 KiB
Python
import asyncio
|
|
import contextlib
|
|
import multiprocessing
|
|
import subprocess
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import Column, Boolean, Integer
|
|
|
|
from wrolpi.common import logger, Base, get_media_directory, get_wrolpi_config
|
|
from wrolpi.vars import DOCKERIZED
|
|
|
|
__all__ = [
|
|
'get_flags',
|
|
'file_worker_busy',
|
|
'refresh_complete',
|
|
]
|
|
|
|
from wrolpi.vars import PYTEST
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
TESTING_LOCK = multiprocessing.Event()
|
|
|
|
FLAG_NAMES = set()
|
|
|
|
|
|
class Flag:
|
|
"""A simple wrapper around multiprocessing.Event.
|
|
|
|
This allows synchronization between the App and this API.
|
|
|
|
This may store its value in the DB table wrolpi_flag."""
|
|
|
|
def __init__(self, name: str, store_db: bool = False):
|
|
self.name = name
|
|
self.store_db = store_db
|
|
FLAG_NAMES.add(name)
|
|
|
|
def __repr__(self):
|
|
return f'<Flag {repr(self.name)}>'
|
|
|
|
def set(self):
|
|
"""Set the multiprocessing.Event for this Flag."""
|
|
if PYTEST and not TESTING_LOCK.is_set():
|
|
# Testing, but the test does not need flags.
|
|
return
|
|
|
|
from wrolpi.api_utils import api_app
|
|
api_app.shared_ctx.flags.update({self.name: True})
|
|
self._save(True)
|
|
|
|
def clear(self):
|
|
"""Clear the multiprocessing.Event for this Flag."""
|
|
if PYTEST and not TESTING_LOCK.is_set():
|
|
# Testing, but the test does not need flags.
|
|
return
|
|
|
|
from wrolpi.api_utils import api_app
|
|
api_app.shared_ctx.flags.update({self.name: False})
|
|
self._save(False)
|
|
|
|
def is_set(self):
|
|
"""Return True if this multiprocessing.Event is set."""
|
|
if PYTEST and not TESTING_LOCK.is_set():
|
|
# Testing, but the test does not need flags.
|
|
return
|
|
|
|
from wrolpi.api_utils import api_app
|
|
return api_app.shared_ctx.flags[self.name]
|
|
|
|
def __enter__(self):
|
|
if PYTEST and not TESTING_LOCK.is_set():
|
|
# Testing, but the test does not need flags.
|
|
return
|
|
|
|
if self.is_set():
|
|
raise ValueError(f'{self} flag is already set!')
|
|
self.set()
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
if PYTEST and not TESTING_LOCK.is_set():
|
|
# Testing, but the test does not need flags.
|
|
return
|
|
|
|
self.clear()
|
|
|
|
def _save(self, value: bool):
|
|
if self.store_db:
|
|
from wrolpi.db import get_db_curs
|
|
# Store the value of this Flag in it's matching column in the DB. Not all flags will do this.
|
|
try:
|
|
with get_db_curs(commit=True) as curs:
|
|
# Upsert the only row in the table.
|
|
curs.execute(f'''
|
|
INSERT INTO wrolpi_flag (id, {self.name}) VALUES (1, %(value)s)
|
|
ON CONFLICT (id) DO UPDATE SET {self.name} = %(value)s
|
|
''', dict(value=value))
|
|
except Exception as e:
|
|
logger.critical(f'Unable to save flag! {repr(self)}', exc_info=e)
|
|
|
|
@contextlib.asynccontextmanager
|
|
async def wait_for(self, timeout: int = 0):
|
|
"""Wait for this Flag to be set."""
|
|
async with wait_for_flag(self, timeout=timeout):
|
|
yield
|
|
|
|
|
|
# Set by `check_db_is_up`.
|
|
db_up = Flag('db_up')
|
|
# Outdated Zims need to be removed.
|
|
outdated_zims = Flag('outdated_zims', store_db=True)
|
|
# Used to disable or enable downloading when Internet is down.
|
|
have_internet = Flag('have_internet')
|
|
# Every configured destination (videos/archive/zims/map) has a mounted drive at or above it.
|
|
# Cleared when at least one destination would write to the unmounted root filesystem.
|
|
media_mounted = Flag('media_mounted')
|
|
|
|
# The FileWorker is busy (refreshing, moving, tagging, or renaming files).
|
|
file_worker_busy = Flag('file_worker_busy')
|
|
# Steps of FileWorker operations.
|
|
file_worker_counting = Flag('file_worker_counting')
|
|
file_worker_discovery = Flag('file_worker_discovery')
|
|
file_worker_modeling = Flag('file_worker_modeling')
|
|
file_worker_indexing = Flag('file_worker_indexing')
|
|
file_worker_cleanup = Flag('file_worker_cleanup')
|
|
|
|
# A global refresh is currently running.
|
|
global_refresh_active = Flag('global_refresh_active')
|
|
|
|
# Map search index is being built.
|
|
map_search_building = Flag('map_search_building')
|
|
|
|
# The global refresh has been performed. This is False on a fresh instance of WROLPi.
|
|
refresh_complete = Flag('refresh_complete', store_db=True)
|
|
|
|
# Cookies flags for yt-dlp.
|
|
cookies_exist = Flag('cookies_exist')
|
|
cookies_unlocked = Flag('cookies_unlocked')
|
|
|
|
|
|
def get_flags() -> dict:
|
|
"""Return a list of all Flags which are set."""
|
|
flags = dict(
|
|
cookies_exist=cookies_exist.is_set(),
|
|
cookies_unlocked=cookies_unlocked.is_set(),
|
|
db_up=db_up.is_set(),
|
|
file_worker_busy=file_worker_busy.is_set(),
|
|
file_worker_cleanup=file_worker_cleanup.is_set(),
|
|
file_worker_counting=file_worker_counting.is_set(),
|
|
file_worker_discovery=file_worker_discovery.is_set(),
|
|
file_worker_indexing=file_worker_indexing.is_set(),
|
|
file_worker_modeling=file_worker_modeling.is_set(),
|
|
global_refresh_active=global_refresh_active.is_set(),
|
|
have_internet=have_internet.is_set(),
|
|
map_search_building=map_search_building.is_set(),
|
|
media_mounted=media_mounted.is_set(),
|
|
outdated_zims=outdated_zims.is_set(),
|
|
refresh_complete=refresh_complete.is_set(),
|
|
)
|
|
return flags
|
|
|
|
|
|
class WROLPiFlag(Base):
|
|
__tablename__ = 'wrolpi_flag'
|
|
id = Column(Integer, primary_key=True)
|
|
refresh_complete = Column(Boolean, default=False)
|
|
outdated_zims = Column(Boolean, default=False)
|
|
|
|
def __repr__(self):
|
|
return f'<WROLPiFlag' \
|
|
f' refresh_complete={self.refresh_complete}' \
|
|
f' outdated_zims={self.outdated_zims}>'
|
|
|
|
|
|
def _destinations_have_mounted_storage() -> bool:
|
|
"""True when every configured destination is backed by a mounted drive.
|
|
|
|
Walks each destination (videos/archive/zims/map) from its static path
|
|
prefix up through the media directory; if any ancestor in that range is
|
|
a mount point, the destination is considered covered. All destinations
|
|
must be covered for this to return True — otherwise a download could
|
|
silently write to the SD card on a Raspberry Pi.
|
|
|
|
Docker mode is exempt: the media directory is always a bind mount and
|
|
the user cannot remount drives from inside the container.
|
|
"""
|
|
if DOCKERIZED:
|
|
return True
|
|
|
|
media_dir = get_media_directory().resolve()
|
|
|
|
config = get_wrolpi_config()
|
|
destinations = [
|
|
config.videos_destination,
|
|
config.archive_destination,
|
|
config.zims_destination,
|
|
config.map_destination,
|
|
]
|
|
|
|
for dest in destinations:
|
|
if not dest:
|
|
continue
|
|
# Templated paths like 'videos/%(channel_tag)s/...' can't be fully
|
|
# resolved without a real channel; use the static prefix.
|
|
static_prefix = dest.split('%', 1)[0].rstrip('/')
|
|
candidate = (media_dir / static_prefix) if static_prefix else media_dir
|
|
# Resolve to normalize '..' segments before the walk; otherwise a
|
|
# destination like '../outside' would still walk back through
|
|
# media_dir and falsely report covered.
|
|
candidate = candidate.resolve()
|
|
|
|
# If the destination escapes the media directory entirely, the
|
|
# download would land outside any drive we control — treat as
|
|
# uncovered so the warning fires.
|
|
if candidate != media_dir and media_dir not in candidate.parents:
|
|
return False
|
|
|
|
covered = False
|
|
cursor = candidate
|
|
while True:
|
|
try:
|
|
if cursor.is_mount():
|
|
covered = True
|
|
break
|
|
except OSError:
|
|
# Path doesn't exist yet (e.g. fresh install); keep walking.
|
|
pass
|
|
if cursor == media_dir:
|
|
break
|
|
cursor = cursor.parent
|
|
if not covered:
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def check_db_is_up():
|
|
"""Attempts to connect to the database, sets flags.db_up if successful."""
|
|
from wrolpi.db import get_db_curs, get_db_args
|
|
|
|
db_args = get_db_args()
|
|
db_host = db_args['host']
|
|
if db_host != '127.0.0.1':
|
|
# Check if database host is up before attempting connection. This will fail faster than SQLAlchemy timeout.
|
|
try:
|
|
cmd = ['ping', '-w', '1', '-c', '1', db_host]
|
|
subprocess.check_call(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
|
|
except Exception as e:
|
|
logger.debug(f'Unable to resolve database host {db_host}', exc_info=e)
|
|
db_up.clear()
|
|
return
|
|
|
|
try:
|
|
with get_db_curs() as curs:
|
|
curs.execute('SELECT * FROM alembic_version')
|
|
# If we get here, the database is up!
|
|
db_up.set()
|
|
except Exception as e:
|
|
logger.debug(f'DB is not up', exc_info=e)
|
|
db_up.clear()
|
|
|
|
|
|
def init_flags():
|
|
"""Read flag values from the DB, copy them to the shared context flags."""
|
|
if not db_up.is_set():
|
|
logger.error(f'Refusing to initialize flags when DB is not up.')
|
|
return
|
|
|
|
from wrolpi.api_utils import api_app
|
|
if api_app.shared_ctx.flags_initialized.is_set():
|
|
# Only need to read from once DB at startup.
|
|
return
|
|
api_app.shared_ctx.flags_initialized.set()
|
|
|
|
logger.debug('Initializing flags...')
|
|
|
|
from wrolpi.db import get_db_session
|
|
with get_db_session() as session:
|
|
flags: WROLPiFlag = session.query(WROLPiFlag).one_or_none()
|
|
if flags:
|
|
if flags.refresh_complete is True:
|
|
refresh_complete.set()
|
|
else:
|
|
refresh_complete.clear()
|
|
if flags.outdated_zims is True:
|
|
outdated_zims.set()
|
|
else:
|
|
outdated_zims.clear()
|
|
|
|
# Initialize cookies flags from cookies module state.
|
|
from modules.videos.cookies import cookies_exist as check_cookies_exist, cookies_unlocked as check_cookies_unlocked
|
|
if check_cookies_exist():
|
|
cookies_exist.set()
|
|
if check_cookies_unlocked():
|
|
cookies_unlocked.set()
|
|
|
|
logger.debug('Initialized flags')
|
|
|
|
|
|
@contextlib.asynccontextmanager
|
|
async def wait_for_flag(flag: Flag, timeout: int = 0):
|
|
"""Sleeps until the provided Flag is set.
|
|
|
|
>>> async with wait_for_flag(db_up):
|
|
>>> do_db_operation()
|
|
|
|
@raise TimeoutError: Raises this when the timeout is reached.
|
|
"""
|
|
start = datetime.now()
|
|
while True:
|
|
if timeout and (elapsed := (datetime.now() - start).total_seconds()) > timeout:
|
|
raise TimeoutError(f'Waited too long ({elapsed}s) for {flag} to be set!')
|
|
|
|
if flag.is_set():
|
|
break
|
|
await asyncio.sleep(0.1)
|
|
|
|
yield
|