diff --git a/.circleci/config.yml b/.circleci/config.yml index 70ad262c..5835c5bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,7 +61,7 @@ jobs: - run: sudo apt-get update - run: sudo apt-get install -y ffmpeg catdoc - restore_cache: - key: deps-3.11-{{ checksum "requirements.txt" }}-2 + key: deps-3.11-{{ checksum "requirements.txt" }}-4 - run: name: Install Requirements command: | @@ -69,7 +69,7 @@ jobs: . venv/bin/activate pip install -r requirements.txt - save_cache: - key: deps-3.11-{{ checksum "requirements.txt" }}-2 + key: deps-3.11-{{ checksum "requirements.txt" }}-4 paths: - "venv" - run: @@ -88,7 +88,7 @@ jobs: steps: - checkout - restore_cache: - key: deps-3.11-{{ checksum "requirements.txt" }}-2 + key: deps-3.11-{{ checksum "requirements.txt" }}-4 - run: name: Install Requirements command: | @@ -96,7 +96,7 @@ jobs: . venv/bin/activate pip install -r requirements.txt - save_cache: - key: deps-3.11-{{ checksum "requirements.txt" }}-2 + key: deps-3.11-{{ checksum "requirements.txt" }}-4 paths: - "venv" - run: @@ -115,7 +115,7 @@ jobs: - run: sudo apt-get update - run: sudo apt-get install -y ffmpeg catdoc - restore_cache: - key: deps-3.12-{{ checksum "requirements.txt" }}-2 + key: deps-3.12-{{ checksum "requirements.txt" }}-4 - run: name: Install Requirements command: | @@ -123,7 +123,7 @@ jobs: . venv/bin/activate pip install -r requirements.txt - save_cache: - key: deps-3.12-{{ checksum "requirements.txt" }}-2 + key: deps-3.12-{{ checksum "requirements.txt" }}-4 paths: - "venv" - run: @@ -134,7 +134,7 @@ jobs: steps: - checkout - restore_cache: - key: deps-3.11-{{ checksum "requirements.txt" }}-2 + key: deps-3.11-{{ checksum "requirements.txt" }}-4 - run: name: Install dependencies command: | diff --git a/alembic/versions/2bfb3b50b178_tag_names_migration.py b/alembic/versions/2bfb3b50b178_tag_names_migration.py index 2e824098..6bd81b19 100644 --- a/alembic/versions/2bfb3b50b178_tag_names_migration.py +++ b/alembic/versions/2bfb3b50b178_tag_names_migration.py @@ -1,4 +1,4 @@ -"""Tag names migration +"""MTag names migration Revision ID: 2bfb3b50b178 Revises: 8d0d81bc9c34 @@ -6,13 +6,21 @@ Create Date: 2024-07-31 14:58:29.583029 """ import os -import sys +import pathlib +from datetime import datetime +import yaml from alembic import op +from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, String +from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Session +from sqlalchemy.orm.exc import NoResultFound # noqa -from wrolpi.common import escape_file_name -from wrolpi.tags import Tag, get_tags_config +from wrolpi import dates +from wrolpi.common import escape_file_name, get_media_directory, ModelHelper +from wrolpi.common import get_relative_to_media_directory +from wrolpi.dates import TZDateTime +from wrolpi.tags import get_tags_config # revision identifiers, used by Alembic. revision = '2bfb3b50b178' @@ -22,13 +30,97 @@ depends_on = None DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False +Base = declarative_base() + + +class MFileGroup(ModelHelper, Base): + __tablename__ = 'file_group' + id: int = Column(BigInteger, primary_key=True) + primary_path: pathlib.Path = Column(String, unique=True, nullable=False) + + +class MTagFile(ModelHelper, Base): + __tablename__ = 'tag_file' + tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True) + file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), primary_key=True) + + +class MTag(ModelHelper, Base): + __tablename__ = 'tag' + id = Column(Integer, primary_key=True) + name = Column(String, unique=True, nullable=False) + color = Column(String) + + +class MZim(Base): + __tablename__ = 'zim' + id = Column(Integer, primary_key=True) + path: pathlib.Path = Column(String, unique=True, nullable=False) + + +class MTagZimEntry(Base): + __tablename__ = 'tag_zim' + tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True) + zim_id = Column(Integer, ForeignKey('zim.id', ondelete='CASCADE'), primary_key=True) + zim_entry: str = Column(Text, nullable=False, primary_key=True) + created_at: datetime = Column(TZDateTime, default=dates.now) + + +def save_tags(session: Session): + media_directory = get_media_directory() + + tags = dict() + tag_rows = session.query(MTag) + for tag in tag_rows: + tags[tag.name] = dict(color=tag.color) + + results = session.query(MTag, MTagFile, MFileGroup) \ + .filter(MTagFile.tag_id == MTag.id, MTagFile.file_group_id == MFileGroup.id) \ + .order_by(MFileGroup.primary_path) + + tag_files = [] + for tag, tag_file, file_group in results: + value = [ + tag.name, + str(file_group.primary_path.relative_to(media_directory)), + # Fallback to current time if not set. + tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(), + ] + tag_files.append(value) + + results = session.query(MTag, MZim, MTagZimEntry) \ + .filter(MTag.id == MTagZimEntry.tag_id, MZim.id == MTagZimEntry.zim_id) \ + .order_by(MTagZimEntry.zim_id, MTagZimEntry.zim_entry) + + tag_zims = [] + for tag, zim, tag_zim_entry in results: + zim: MZim + tag_zim_entry: MTagZimEntry + value = [ + tag.name, + str(get_relative_to_media_directory(zim.path)), + tag_zim_entry.zim_entry, + # Fallback to current time if not set. + tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(), + ] + tag_zims.append(value) + + # Write to the config. + with get_tags_config().get_file().open('wt') as fh: + data = { + 'tag_files': tag_files, + 'tag_zims': tag_zims, + 'tags': tags, + } + yaml.dump(data, fh) + def upgrade(): bind = op.get_bind() session = Session(bind=bind) need_commit = False - tags = session.query(Tag).all() + tags = session.query(MTag).all() for tag in tags: escaped_name = escape_file_name(tag.name) if escaped_name != tag.name: @@ -38,13 +130,7 @@ def upgrade(): if need_commit: session.commit() - - try: - config = get_tags_config() - config.initialize() - config.save_tags(session, ignore_lock=True) - except FileNotFoundError: - print(f'Failed to save tags config probably because media directory does not exist', file=sys.stderr) + save_tags(session) def downgrade(): diff --git a/app/src/Tags.js b/app/src/Tags.js index 2e7736b1..3e5759b8 100644 --- a/app/src/Tags.js +++ b/app/src/Tags.js @@ -1,6 +1,6 @@ import React, {useEffect} from "react"; import {deleteTag, getTags, saveTag} from "./api"; -import {Dimmer, Divider, Form, Grid, GridColumn, GridRow, Label, TableCell, TableRow,} from "semantic-ui-react"; +import {Dimmer, Divider, Form, Grid, GridColumn, GridRow, Input, Label, TableCell, TableRow,} from "semantic-ui-react"; import {APIButton, contrastingColor, ErrorMessage, fuzzyMatch, scrollToTopOfElement} from "./components/Common"; import { Button, diff --git a/main.py b/main.py index a8aa178b..e0ee085e 100755 --- a/main.py +++ b/main.py @@ -11,13 +11,12 @@ from sanic.signals import Event from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS, admin from wrolpi import root_api # noqa from wrolpi import tags -from wrolpi.api_utils import api_app +from wrolpi.api_utils import api_app, perpetual_signal from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \ cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts from wrolpi.dates import Seconds -from wrolpi.downloader import import_downloads_config, download_manager, \ - perpetual_download_worker +from wrolpi.downloader import import_downloads_config, download_manager from wrolpi.vars import PROJECT_DIR, DOCKERIZED, INTERNET_SERVER from wrolpi.version import get_version_string @@ -145,7 +144,7 @@ def main(): return 1 -@api_app.main_process_start +@api_app.main_process_ready async def startup(app: Sanic): """ Initializes multiprocessing tools, flags, etc. @@ -167,7 +166,7 @@ async def startup(app: Sanic): logger.debug('startup done') -@api_app.listener('before_server_start') # FileConfigs need to be initialized first. +@api_app.listener('after_server_start') # FileConfigs need to be initialized first. async def initialize_configs(app: Sanic): """Each Sanic process runs this once.""" # Each process will have their own FileConfig object, but share the `app.shared_ctx.*config` @@ -177,13 +176,11 @@ async def initialize_configs(app: Sanic): initialize_configs_contexts(app) await asyncio.sleep(0.5) wrol_mode_enabled() - logger.info('initialize_configs succeeded') + logger.info(f'initialize_configs succeeded pid={os.getpid()}') except Exception as e: logger.error('initialize_configs failed with', exc_info=e) raise - logger.debug('initialize_configs done') - @api_app.signal(Event.SERVER_SHUTDOWN_BEFORE) @api_app.listener('reload_process_stop') @@ -203,7 +200,7 @@ async def handle_server_shutdown_reset(app: Sanic, loop): # Start periodic tasks after configs are ready. -@api_app.listener('after_server_start') +@api_app.after_server_start async def start_single_tasks(app: Sanic): """Recurring/Single tasks that are started in only one Sanic process.""" # Only allow one child process to perform periodic tasks. See `handle_server_shutdown` @@ -219,14 +216,6 @@ async def start_single_tasks(app: Sanic): else: download_manager.disable() - # Start perpetual tasks. DO NOT AWAIT! - app.add_task(perpetual_check_db_is_up_worker()) # noqa - app.add_task(perpetual_download_worker()) # noqa - app.add_task(perpetual_have_internet_worker()) # noqa - - await app.dispatch('wrolpi.periodic.start_video_missing_comments_download') - await app.dispatch('wrolpi.periodic.status') - try: flag_outdated_zim_files() except Exception as e: @@ -275,14 +264,7 @@ async def start_single_tasks(app: Sanic): logger.debug(f'start_single_tasks done') -@api_app.listener('after_server_start') -async def start_sanic_worker(app: Sanic): - logger.debug(f'start_sanic_worker') - - await app.dispatch('wrolpi.periodic.check_log_level') - - -@api_app.signal('wrolpi.periodic.check_log_level') +@perpetual_signal(sleep=1) async def periodic_check_log_level(): """Copies global log level into this Sanic worker's logger.""" log_level = api_app.shared_ctx.log_level.value @@ -290,49 +272,31 @@ async def periodic_check_log_level(): logger.info(f'changing log level from {logger.getEffectiveLevel()} to {log_level}') set_log_level(log_level, warn_level=False) - try: - await asyncio.sleep(1) - await api_app.dispatch('wrolpi.periodic.check_log_level') - except asyncio.CancelledError: - # Server is shutting down. - pass - +@perpetual_signal(sleep=10) async def perpetual_check_db_is_up_worker(): try: - while True: - try: - flags.check_db_is_up() - flags.init_flags() - except Exception as e: - logger.error('Failed to check if db is up', exc_info=e) - - await asyncio.sleep(10) - except asyncio.CancelledError: - logger.info('periodic_check_db_is_up was cancelled...') + flags.check_db_is_up() + flags.init_flags() except Exception as e: logger.error('Failed to check db status', exc_info=e) +@perpetual_signal(sleep=10) async def perpetual_have_internet_worker(): try: - while True: - try: - if can_connect_to_server(INTERNET_SERVER): - flags.have_internet.set() - # Check hourly once we have internet. - await asyncio.sleep(float(Seconds.hour)) - else: - # Check more often until the internet is back. - flags.have_internet.clear() - await asyncio.sleep(10) - except Exception as e: - logger.error('Failed to check if internet is up', exc_info=e) - except asyncio.CancelledError: - logger.info('perpetual_check_have_internet_worker was cancelled...') + if can_connect_to_server(INTERNET_SERVER): + flags.have_internet.set() + # Check hourly once we have internet. + await asyncio.sleep(float(Seconds.hour)) + else: + # Check more often until the internet is back. + flags.have_internet.clear() + except Exception as e: + logger.error('Failed to check if internet is up', exc_info=e) -@api_app.signal('wrolpi.periodic.start_video_missing_comments_download') +@perpetual_signal(sleep=30) async def periodic_start_video_missing_comments_download(): from modules.videos.video.lib import get_missing_videos_comments @@ -358,8 +322,6 @@ async def periodic_start_video_missing_comments_download(): await asyncio.sleep(int(Seconds.hour)) - await api_app.dispatch('wrolpi.periodic.start_video_missing_comments_download') - if __name__ == '__main__': sys.exit(main()) diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index 0af2e71e..4defc429 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -82,20 +82,20 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_ @pytest.mark.asyncio -async def test_search_archive_tags(test_session, test_async_client, archive_factory, tag_factory): +async def test_search_archive_tags(test_session, async_client, archive_factory, tag_factory): """Tagged Archives can be searched.""" tag = await tag_factory() archive_factory(domain='example.com', tag_names=[tag.name, ]) test_session.commit() content = {'search_str': '', 'domain': 'example.com', 'tag_names': [tag.name, ]} - request, response = await test_async_client.post('/api/archive/search', content=json.dumps(content)) + request, response = await async_client.post('/api/archive/search', content=json.dumps(content)) assert response.status_code == HTTPStatus.OK assert response.json['file_groups'] assert response.json['totals']['file_groups'] == 1 content = {'search_str': '', 'domain': 'example.com', 'tag_names': ['does not exist', ]} - request, response = await test_async_client.post('/api/archive/search', content=json.dumps(content)) + request, response = await async_client.post('/api/archive/search', content=json.dumps(content)) assert response.status_code == HTTPStatus.OK assert response.json['file_groups'] == [] assert response.json['totals']['file_groups'] == 0 diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index ceb254b6..bd3ae8f7 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -80,7 +80,7 @@ async def test_relationships(test_session, example_singlefile): @pytest.mark.asyncio -async def test_archive_title(test_async_client, test_session, archive_factory, singlefile_contents_factory): +async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory): """An Archive's title can be fetched in multiple ways. This tests from most to least reliable.""" # Create some test files, delete all records for a fresh refresh. archive_factory( @@ -459,7 +459,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b @pytest.mark.asyncio -async def test_refresh_archives(test_session, test_directory, test_async_client, make_files_structure): +async def test_refresh_archives(test_session, test_directory, async_client, make_files_structure): """Archives can be found and put in the database. A single Archive will have multiple files.""" # The start of a typical singlefile html file. singlefile_contents = '''