diff --git a/alembic/versions/2026_10_08_1200_file_group_progress.py b/alembic/versions/2026_10_08_1200_file_group_progress.py new file mode 100644 index 00000000..cc7e6f3e --- /dev/null +++ b/alembic/versions/2026_10_08_1200_file_group_progress.py @@ -0,0 +1,24 @@ +"""A FileGroup records how far the user is through it: file_group.progress and file_group.position. + +Revision ID: 2026_10_08_1200 +Revises: 2026_10_06_1500 +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_10_08_1200' +down_revision = '2026_10_06_1500' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('file_group', sa.Column('progress', sa.Float(), nullable=True)) + op.add_column('file_group', sa.Column('position', sa.JSON(), nullable=True)) + + +def downgrade(): + with op.batch_alter_table('file_group') as batch_op: + batch_op.drop_column('position') + batch_op.drop_column('progress') diff --git a/wrolpi/common.py b/wrolpi/common.py index 11bd80c9..7a6aa83b 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1190,6 +1190,10 @@ def get_all_configs() -> Dict[str, ConfigFile]: if repos_config := get_repos_config(): all_configs[repos_config.file_name] = repos_config + from wrolpi.files.progress import get_recently_viewed_config + if recently_viewed_config := get_recently_viewed_config(): + all_configs[recently_viewed_config.file_name] = recently_viewed_config + return all_configs @@ -1316,6 +1320,16 @@ async def import_all_db_configs() -> dict[str, bool]: logger.warning(f'Failed to import flasher config: {e}') results['flasher'] = False + # Recently viewed files (positions in FileGroups, no dependencies) + try: + from wrolpi.files.progress import get_recently_viewed_config + await asyncio.to_thread(get_recently_viewed_config().import_config) + results['recently_viewed'] = True + logger.debug('recently viewed config imported') + except Exception as e: + logger.warning(f'Failed to import recently viewed config: {e}') + results['recently_viewed'] = False + return results diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index e829f646..55559427 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -39,6 +39,7 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.map_pins_config = manager.dict() app.shared_ctx.flasher_config = manager.dict() app.shared_ctx.bookmarks_config = manager.dict() + app.shared_ctx.recently_viewed_config = manager.dict() # Shared dicts. app.shared_ctx.uploaded_files = manager.dict() app.shared_ctx.status = manager.dict() @@ -332,3 +333,9 @@ def initialize_configs_contexts(app: Sanic): BOOKMARKS_CONFIG.initialize(app.shared_ctx.bookmarks_config) except Exception as e: logger.error(f'Failed to initialize in-memory bookmarks config: {e}') + + try: + from wrolpi.files.progress import RECENTLY_VIEWED_CONFIG + RECENTLY_VIEWED_CONFIG.initialize(app.shared_ctx.recently_viewed_config) + except Exception as e: + logger.error(f'Failed to initialize in-memory recently viewed config: {e}') diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index dad326cb..be7f89c1 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -57,6 +57,29 @@ async def get_file(request: Request, body: schema.FileRequest): return json_response({'file': file}) +@files_bp.post('/progress') +@openapi.definition( + summary='Record how far the user is through a file, and where to resume it', + body=schema.FileProgressRequest, +) +@validate(schema.FileProgressRequest) +async def post_file_progress(request: Request, body: schema.FileProgressRequest): + path = get_media_directory() / body.file + progress, position = lib.set_file_progress(request.ctx.session, path, body.progress, body.position, body.final) + return json_response({'progress': progress, 'position': position}) + + +@files_bp.post('/progress/clear') +@openapi.definition( + summary='Forget how far the user is through a file, so it starts from the beginning', + body=schema.FileProgressClearRequest, +) +@validate(schema.FileProgressClearRequest) +async def post_file_progress_clear(request: Request, body: schema.FileProgressClearRequest): + lib.clear_file_progress(request.ctx.session, get_media_directory() / body.file) + return response.empty(HTTPStatus.NO_CONTENT) + + @files_bp.post('/delete') @openapi.definition( summary='Delete files or directories. Directories are deleted recursively.' diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 61e620d3..ee7eb071 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -36,6 +36,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError, UnsupportedArchive, InvalidArchiveMember from wrolpi.events import Events from wrolpi.files.models import FileGroup, Directory +from wrolpi.files.progress import normalize_progress, save_recently_viewed_config from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config, sync_tags_directory from wrolpi.vars import PYTEST, IS_MACOS @@ -124,11 +125,14 @@ def _get_file_dict(session: Session, file: pathlib.Path) -> Dict: mimetype = get_mimetype(file) except PermissionError: mimetype = None + file_group = FileGroup.get_by_path(session, file) return dict( path=file.relative_to(media_directory), size=size, mimetype=mimetype, tags=get_file_tag_names(session, file), + progress=file_group.progress if file_group else None, + position=file_group.position if file_group else None, ) @@ -156,6 +160,53 @@ async def set_file_viewed(session: Session, file: pathlib.Path): session.commit() +def _get_or_create_file_group(session: Session, file: pathlib.Path) -> FileGroup: + if not get_paths_in_media_directory([file]): + raise InvalidFile('File must be in the media directory') + try: + return FileGroup.find_by_path(session, file) + except UnknownFile: + if not file.is_file(): + raise InvalidFile(f'File does not exist: {get_relative_to_media_directory(file)}') + fg = FileGroup.from_paths(session, file) + fg.do_model(session) + return fg + + +def set_file_progress(session: Session, file: pathlib.Path, progress: float, position: dict = None, + final: bool = False) -> Tuple[float | None, dict | None]: + """Record how far the User is through a file (see wrolpi.files.progress). This is also a view. + + @param final: The User stopped (paused, closed); save the recently viewed config. + @return: The (progress, position) that was stored. + """ + progress, position = normalize_progress(progress, position) + + # Files in ignored directories are not tracked. + if not remove_files_in_ignored_directories([file]): + return None, None + + fg = _get_or_create_file_group(session, file) + # Not `set_viewed`, which saves the config; a heartbeat during playback only writes the DB. + fg.viewed = now() + fg.progress = progress + fg.position = position + session.commit() + + if final: + save_recently_viewed_config.activate_switch() + return progress, position + + +def clear_file_progress(session: Session, file: pathlib.Path): + """Forget how far the User is through a file, so it starts from the beginning. It remains viewed.""" + fg = _get_or_create_file_group(session, file) + fg.progress = None + fg.position = None + session.commit() + save_recently_viewed_config.activate_switch() + + @cachetools.func.ttl_cache(10_000, 30.0) def _get_directory_dict(directory: pathlib.Path, directories_cache: str, # Used to cache by requested directories. diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index a81e9c44..35eb7133 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -6,7 +6,7 @@ import urllib.parse from datetime import datetime from typing import List, Type, Optional, Iterable -from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON, or_ +from sqlalchemy import Column, String, BigInteger, Boolean, event, Float, Index, Integer, JSON, or_ from sqlalchemy import types from sqlalchemy.orm import deferred, relationship, Session, selectinload @@ -18,6 +18,7 @@ from wrolpi.downloader import Download from wrolpi.errors import FileGroupIsTagged, UnknownFile from wrolpi.files import indexers from wrolpi.media_path import MediaPathType +from wrolpi.files.progress import save_recently_viewed_config from wrolpi.tags import Tag, TagFile, save_tags_config, sync_tags_directory from wrolpi.vars import PYTEST @@ -116,6 +117,9 @@ class FileGroup(ModelHelper, Base): title = Column(String) # user-displayable title url = Column(String) # the location where this file can be downloaded. viewed = Column(TZDateTime) # the most recent time a User viewed this file. + # How far the User is through this file, see wrolpi.files.progress. + progress = Column(Float) # 0.0 to 1.0, where 1.0 is finished. + position = Column(JSON) # Where to resume, e.g. {"kind": "time", "seconds": 512.3}. # Columns updated by triggers. # `file_group_effective_datetime_trigger` and `update_effective_datetime` event handler @@ -180,6 +184,8 @@ class FileGroup(ModelHelper, Base): title=self.title, url=self.url, viewed=self.viewed, + progress=self.progress, + position=self.position, ) return d @@ -224,6 +230,7 @@ class FileGroup(ModelHelper, Base): :return: The datetime that was set. """ self.viewed = viewed or now() + save_recently_viewed_config.activate_switch() return self.viewed def set_tags(self, session: Session, tag_names_or_ids: Iterable[str | int]): diff --git a/wrolpi/files/progress.py b/wrolpi/files/progress.py new file mode 100644 index 00000000..1028c0e2 --- /dev/null +++ b/wrolpi/files/progress.py @@ -0,0 +1,238 @@ +"""How far the User is through a file, and the recently viewed files config. + +`FileGroup.progress` is the fraction consumed (0.0 to 1.0, where 1.0 is finished), and `FileGroup.position` +is where to resume. A position is a dict with a `kind`: + + {"kind": "time", "seconds": 512.3} video and audio + {"kind": "epub", "cfi": "epubcfi(/6/14!/4/2)"} EPUB + {"kind": "page", "page": 12} comic books (0-based) + +The client may add `updated_at` (milliseconds since the epoch) so it can compare the server's position with +one it holds locally. + +The database is the working store. `recently_viewed.yaml` holds the most recently viewed files so viewing +history and positions survive a database rebuild. +""" +import math +from dataclasses import dataclass, field +from datetime import datetime +from typing import List, Optional, Tuple + +import pytz + +from wrolpi import dates +from wrolpi.common import ConfigFile, logger, get_media_directory +from wrolpi.db import get_db_session +from wrolpi.errors import ValidationError, NoPrimaryFile +from wrolpi.events import Events +from wrolpi.switches import register_switch_handler, ActivateSwitchMethod + +logger = logger.getChild(__name__) + +# A file this far through is finished; the rest is credits, an end screen, or an index. +FINISHED_PROGRESS = 0.95 +# A position this close to the start is not worth resuming. +MINIMUM_SECONDS = 10 +MINIMUM_PROGRESS = 0.01 +MAXIMUM_CFI_LENGTH = 1_000 + +# The most recently viewed files kept in the config. The database keeps every `viewed`. +RECENTLY_VIEWED_LIMIT = 1_000 + + +def _number(value, name: str) -> float: + # bool is an int; reject it so `true` is not a position. + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ValidationError(f'{name} must be a number') + return value + + +def validate_position(position: Optional[dict]) -> Optional[dict]: + """Return a clean copy of `position`, or raise ValidationError.""" + if position is None: + return None + if not isinstance(position, dict): + raise ValidationError('position must be an object') + + kind = position.get('kind') + if kind == 'time': + clean = dict(kind=kind, seconds=float(_number(position.get('seconds'), 'seconds'))) + if clean['seconds'] < 0: + raise ValidationError('seconds cannot be negative') + elif kind == 'page': + page = position.get('page') + if isinstance(page, bool) or not isinstance(page, int) or page < 0: + raise ValidationError('page must be a non-negative integer') + clean = dict(kind=kind, page=page) + elif kind == 'epub': + cfi = position.get('cfi') + if not isinstance(cfi, str) or not cfi.startswith('epubcfi(') or len(cfi) > MAXIMUM_CFI_LENGTH: + raise ValidationError('cfi must be an epubcfi') + clean = dict(kind=kind, cfi=cfi) + else: + raise ValidationError(f'Unknown position kind: {kind!r}') + + if (updated_at := position.get('updated_at')) is not None: + clean['updated_at'] = int(_number(updated_at, 'updated_at')) + return clean + + +def normalize_progress(progress, position: Optional[dict]) -> Tuple[float, Optional[dict]]: + """Apply the progress rules: a finished file has no position, nor does one barely started. + + @return: (progress, position) to store. + """ + progress = min(max(float(_number(progress, 'progress')), 0.0), 1.0) + position = validate_position(position) + + if progress >= FINISHED_PROGRESS: + return 1.0, None + + too_early = progress < MINIMUM_PROGRESS + if position and position['kind'] == 'time': + too_early = too_early or position['seconds'] < MINIMUM_SECONDS + elif position and position['kind'] == 'page': + too_early = too_early or position['page'] == 0 + if too_early: + return 0.0, None + + return progress, position + + +@dataclass +class RecentlyViewedConfigValidator: + version: int = None + files: list = field(default_factory=list) + + +def _parse_entry(entry) -> Optional[Tuple[str, datetime, Optional[float], Optional[dict]]]: + """Return (path, viewed, progress, position) from a config entry, or None if it is malformed.""" + if not isinstance(entry, dict) or not isinstance(entry.get('path'), str) or not entry.get('viewed'): + return None + try: + viewed = entry['viewed'] + if isinstance(viewed, datetime): + # YAML parses an unquoted timestamp (a hand edit) into a naive UTC datetime. + viewed = viewed if viewed.tzinfo else viewed.replace(tzinfo=pytz.UTC) + else: + viewed = dates.strptime_ms(viewed) + progress, position = None, None + if entry.get('progress') is not None: + progress, position = normalize_progress(entry['progress'], entry.get('position')) + return entry['path'], viewed, progress, position + except Exception as e: + logger.warning(f'Ignoring invalid recently viewed entry {entry!r}: {e}') + return None + + +class RecentlyViewedConfig(ConfigFile): + file_name = 'recently_viewed.yaml' + default_config = dict( + version=0, + files=[], + ) + validator = RecentlyViewedConfigValidator + width = 500 + + @property + def files(self) -> List[dict]: + return list(self._config.get('files', [])) + + def dump_config(self, file=None, send_events=False, overwrite=False): + if self.get_file().exists() and not self.successful_import and not overwrite: + # Dumping now would replace history the import has not yet applied. + logger.debug(f'Not dumping {self.file_name} because it has not been imported') + return + + try: + # wrolpi.files.models imports this module. + from wrolpi.files.models import FileGroup + media_directory = get_media_directory() + with get_db_session() as session: + file_groups = session.query(FileGroup) \ + .filter(FileGroup.viewed.isnot(None)) \ + .order_by(FileGroup.viewed.desc(), FileGroup.id) \ + .limit(RECENTLY_VIEWED_LIMIT) + files = list() + for file_group in file_groups: + entry = dict( + path=str(file_group.primary_path.relative_to(media_directory)), + viewed=file_group.viewed.isoformat(), + ) + if file_group.progress is not None: + entry['progress'] = file_group.progress + if file_group.position: + entry['position'] = file_group.position + files.append(entry) + self.update({'files': files}, overwrite=overwrite) + except Exception as e: + message = f'Failed to save {self.get_relative_file()} config' + logger.error(message, exc_info=e) + if send_events: + Events.send_config_save_failed(message) + + def import_config(self, file=None, send_events=False): + """Apply the config to the database. The newer `viewed` wins, so a file viewed while WROLPi was + starting keeps its newer position.""" + super().import_config(file, send_events) + try: + # wrolpi.files.models imports this module. + from wrolpi.files.lib import glob_shared_stem + from wrolpi.files.models import FileGroup + + media_directory = get_media_directory() + entries = [i for i in map(_parse_entry, self.files) if i] + with get_db_session(commit=True) as session: + absolute_paths = [media_directory / path for path, *_ in entries] + file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(absolute_paths)) + file_groups_by_path = {i.primary_path: i for i in file_groups} + + applied = 0 + for path, viewed, progress, position in entries: + absolute_path = media_directory / path + file_group = file_groups_by_path.get(absolute_path) + if not file_group: + if not absolute_path.is_file(): + # The file was deleted; its history goes with it. + logger.debug(f'Not importing recently viewed file which does not exist: {path}') + continue + try: + file_group = FileGroup.from_paths(session, *glob_shared_stem(absolute_path)) + except NoPrimaryFile: + logger.warning(f'Cannot import recently viewed file: {path}') + continue + session.add(file_group) + session.flush([file_group]) + file_groups_by_path[absolute_path] = file_group + + if file_group.viewed and file_group.viewed >= viewed: + continue + file_group.viewed = viewed + file_group.progress = progress + file_group.position = position + applied += 1 + + logger.info(f'Imported {applied} of {len(entries)} recently viewed files') + self.successful_import = True + except Exception as e: + self.successful_import = False + message = f'Failed to import {self.get_relative_file()} config' + logger.error(message, exc_info=e) + if send_events: + Events.send_config_import_failed(message) + raise + + +RECENTLY_VIEWED_CONFIG: RecentlyViewedConfig = RecentlyViewedConfig() + + +def get_recently_viewed_config() -> RecentlyViewedConfig: + return RECENTLY_VIEWED_CONFIG + + +@register_switch_handler('save_recently_viewed_config') +def save_recently_viewed_config(): + get_recently_viewed_config().dump_config() + + +save_recently_viewed_config: ActivateSwitchMethod diff --git a/wrolpi/files/schema.py b/wrolpi/files/schema.py index 0eac4945..3322f2dc 100644 --- a/wrolpi/files/schema.py +++ b/wrolpi/files/schema.py @@ -15,6 +15,19 @@ class FileRequest: skip_tracking: bool = False +@dataclass +class FileProgressRequest: + file: str + progress: float + position: Optional[dict] = None + final: bool = False + + +@dataclass +class FileProgressClearRequest: + file: str + + @dataclass class DeleteRequest: paths: List[str] = field(default_factory=list) diff --git a/wrolpi/files/test/test_progress.py b/wrolpi/files/test/test_progress.py new file mode 100644 index 00000000..d5a0369a --- /dev/null +++ b/wrolpi/files/test/test_progress.py @@ -0,0 +1,303 @@ +"""Tests for file progress (how far the user is through a file) and recently_viewed.yaml.""" +import json +from datetime import timedelta +from http import HTTPStatus + +import pytest +import yaml + +from wrolpi import dates +from wrolpi.errors import ValidationError +from wrolpi.files.models import FileGroup +from wrolpi.files.progress import normalize_progress, get_recently_viewed_config, RECENTLY_VIEWED_LIMIT +from wrolpi.files.worker import file_worker, FileTask, FileTaskType + + +def read_recently_viewed(test_directory) -> list: + with (test_directory / 'config/recently_viewed.yaml').open() as fh: + return yaml.safe_load(fh)['files'] + + +@pytest.mark.parametrize('progress,position,expected', [ + # A video part way through keeps its position. + (0.5, dict(kind='time', seconds=300.0), (0.5, dict(kind='time', seconds=300.0))), + # Finished files have no position. + (0.95, dict(kind='time', seconds=570), (1.0, None)), + (1.5, dict(kind='page', page=40), (1.0, None)), + # Barely started files are not worth resuming. + (0.2, dict(kind='time', seconds=9.9), (0.0, None)), + (0.005, dict(kind='epub', cfi='epubcfi(/6/2!/4/2)'), (0.0, None)), + (0.1, dict(kind='page', page=0), (0.0, None)), + (-1, None, (0.0, None)), + # Pages and CFIs. + (0.3, dict(kind='page', page=12), (0.3, dict(kind='page', page=12))), + (0.3, dict(kind='epub', cfi='epubcfi(/6/14!/4/2)', updated_at=1760000000000), + (0.3, dict(kind='epub', cfi='epubcfi(/6/14!/4/2)', updated_at=1760000000000))), + # Unknown keys are dropped. + (0.3, dict(kind='page', page=3, evil='x'), (0.3, dict(kind='page', page=3))), +]) +def test_normalize_progress(progress, position, expected): + assert normalize_progress(progress, position) == expected + + +@pytest.mark.parametrize('progress,position', [ + ('half', None), + (True, None), + (float('nan'), None), + (0.5, dict(kind='time', seconds='300')), + (0.5, dict(kind='time', seconds=-1)), + (0.5, dict(kind='page', page=1.5)), + (0.5, dict(kind='page', page=True)), + (0.5, dict(kind='epub', cfi='javascript:alert(1)')), + (0.5, dict(kind='epub', cfi='epubcfi(' + 'a' * 1_000 + ')')), + (0.5, dict(kind='scroll', fraction=0.5)), + (0.5, 'time'), +]) +def test_normalize_progress_invalid(progress, position): + with pytest.raises(ValidationError): + normalize_progress(progress, position) + + +@pytest.mark.asyncio +async def test_progress_api(test_session, async_client, test_directory, make_files_structure): + """Progress is stored on the FileGroup, returned with the file, and is a view.""" + make_files_structure(['videos/movie.mp4']) + + body = dict(file='videos/movie.mp4', progress=0.4, position=dict(kind='time', seconds=480.5)) + request, response = await async_client.post('/api/files/progress', content=json.dumps(body)) + assert response.status_code == HTTPStatus.OK + assert response.json == dict(progress=0.4, position=dict(kind='time', seconds=480.5)) + + fg, = test_session.query(FileGroup).all() + assert fg.viewed + assert fg.progress == 0.4 + assert fg.position == dict(kind='time', seconds=480.5) + + # The next request sees what was stored. + request, response = await async_client.post('/api/files/file', content=json.dumps( + dict(file='videos/movie.mp4', skip_tracking=True))) + assert response.json['file']['progress'] == 0.4 + assert response.json['file']['position'] == dict(kind='time', seconds=480.5) + + # Rewinding is allowed; the last write wins. + body = dict(file='videos/movie.mp4', progress=0.2, position=dict(kind='time', seconds=240)) + request, response = await async_client.post('/api/files/progress', content=json.dumps(body)) + test_session.expire_all() + assert fg.progress == 0.2 + assert fg.position == dict(kind='time', seconds=240) + + # Start over. + request, response = await async_client.post('/api/files/progress/clear', + content=json.dumps(dict(file='videos/movie.mp4'))) + assert response.status_code == HTTPStatus.NO_CONTENT + test_session.expire_all() + assert fg.progress is None + assert fg.position is None + assert fg.viewed, 'A file that starts over has still been viewed' + + +@pytest.mark.asyncio +async def test_progress_api_errors(test_session, async_client, test_directory, make_files_structure): + make_files_structure(['videos/movie.mp4']) + + body = dict(file='videos/movie.mp4', progress=0.4, position=dict(kind='scroll', fraction=0.4)) + request, response = await async_client.post('/api/files/progress', content=json.dumps(body)) + assert response.status_code == HTTPStatus.BAD_REQUEST + + body = dict(file='videos/missing.mp4', progress=0.4, position=dict(kind='time', seconds=40)) + request, response = await async_client.post('/api/files/progress', content=json.dumps(body)) + assert response.status_code == HTTPStatus.BAD_REQUEST + + # A file outside the media directory. + outside = test_directory.parent / 'outside.mp4' + outside.touch() + try: + for file in ('../outside.mp4', str(outside)): + body = dict(file=file, progress=0.4, position=dict(kind='time', seconds=40)) + request, response = await async_client.post('/api/files/progress', content=json.dumps(body)) + assert response.status_code == HTTPStatus.BAD_REQUEST + request, response = await async_client.post('/api/files/progress/clear', + content=json.dumps(dict(file=file))) + assert response.status_code == HTTPStatus.BAD_REQUEST + finally: + outside.unlink() + + assert test_session.query(FileGroup).count() == 0 + + +@pytest.mark.asyncio +async def test_progress_ignored_directory(test_session, async_client, test_directory, make_files_structure, + test_wrolpi_config, await_switches): + """Files in ignored directories are not tracked.""" + from wrolpi.common import get_wrolpi_config + make_files_structure(['private/movie.mp4']) + get_wrolpi_config().ignored_directories = [str(test_directory / 'private')] + await await_switches() + + body = dict(file='private/movie.mp4', progress=0.4, position=dict(kind='time', seconds=480)) + request, response = await async_client.post('/api/files/progress', content=json.dumps(body)) + assert response.status_code == HTTPStatus.OK + assert test_session.query(FileGroup).count() == 0 + + +@pytest.mark.asyncio +async def test_progress_saves_config_when_final(test_session, async_client, test_directory, make_files_structure): + """Only a final progress (pause, close) saves the config; a heartbeat during playback writes only the DB.""" + make_files_structure(['videos/movie.mp4']) + config_file = test_directory / 'config/recently_viewed.yaml' + + body = dict(file='videos/movie.mp4', progress=0.4, position=dict(kind='time', seconds=480)) + await async_client.post('/api/files/progress', content=json.dumps(body)) + assert not config_file.exists() + + body['final'] = True + await async_client.post('/api/files/progress', content=json.dumps(body)) + entry, = read_recently_viewed(test_directory) + assert entry['path'] == 'videos/movie.mp4' + assert entry['progress'] == 0.4 + assert entry['position'] == dict(kind='time', seconds=480.0) + assert dates.strptime_ms(entry['viewed']) + + # Starting over saves the config. + await async_client.post('/api/files/progress/clear', content=json.dumps(dict(file='videos/movie.mp4'))) + entry, = read_recently_viewed(test_directory) + assert 'progress' not in entry and 'position' not in entry + + +@pytest.mark.asyncio +async def test_view_saves_config(test_session, async_client, test_directory, make_files_structure, refresh_files, + await_background_tasks, await_switches): + """Opening a file saves it in the recently viewed config.""" + make_files_structure({'docs/notes.txt': 'notes'}) + await refresh_files() + await await_switches() + # The refresh saved the config, with nothing viewed yet. + assert read_recently_viewed(test_directory) == [] + + await async_client.post('/api/files/file', content=json.dumps(dict(file='docs/notes.txt'))) + # Viewing a file is a background task, which activates the switch. + await await_background_tasks() + await await_switches() + + entry, = read_recently_viewed(test_directory) + assert entry['path'] == 'docs/notes.txt' + assert 'progress' not in entry + + +@pytest.mark.asyncio +async def test_recently_viewed_dump_order_and_limit(test_session, async_client, test_directory, make_files_structure, + monkeypatch): + """The config holds the most recently viewed files, newest first.""" + monkeypatch.setattr('wrolpi.files.progress.RECENTLY_VIEWED_LIMIT', 2) + old, middle, new, never = make_files_structure(['old.txt', 'middle.txt', 'new.txt', 'never.txt']) + now = dates.now() + for path, viewed in ((old, now - timedelta(days=2)), (middle, now - timedelta(days=1)), (new, now), (never, None)): + fg = FileGroup.from_paths(test_session, path) + fg.viewed = viewed + test_session.commit() + + config = get_recently_viewed_config() + config.dump_config() + config.save() + + assert [i['path'] for i in read_recently_viewed(test_directory)] == ['new.txt', 'middle.txt'] + assert RECENTLY_VIEWED_LIMIT == 1_000 + + +@pytest.mark.asyncio +async def test_recently_viewed_import(test_session, async_client, test_directory, make_files_structure, + video_file_factory): + """A wiped database gets its viewing history back from the config. The newer `viewed` wins.""" + _, book, newer = make_files_structure(['videos/movie.en.vtt', 'books/book.epub', 'videos/newer.mp4']) + video_file_factory(test_directory / 'videos/movie.mp4') + now = dates.now() + + # This file was viewed after the config was written (e.g. while WROLPi was starting). + newer_fg = FileGroup.from_paths(test_session, newer) + newer_fg.viewed = now + newer_fg.progress = 0.8 + newer_fg.position = dict(kind='time', seconds=800.0) + test_session.commit() + + config_file = test_directory / 'config/recently_viewed.yaml' + config_file.parent.mkdir(exist_ok=True) + config_file.write_text(yaml.dump(dict(version=3, files=[ + dict(path='videos/newer.mp4', viewed=(now - timedelta(hours=1)).isoformat(), progress=0.1, + position=dict(kind='time', seconds=100)), + dict(path='videos/movie.mp4', viewed=(now - timedelta(hours=2)).isoformat(), progress=0.5, + position=dict(kind='time', seconds=600)), + dict(path='books/book.epub', viewed=(now - timedelta(hours=3)).isoformat(), progress=1.0), + # The file was deleted. + dict(path='videos/deleted.mp4', viewed=(now - timedelta(hours=4)).isoformat(), progress=0.5), + # Invalid entries are ignored. + dict(path='videos/movie.mp4', viewed='not a date'), + dict(path='books/book.epub', viewed=now.isoformat(), progress=0.5, position=dict(kind='scroll')), + 'not an entry', + ]))) + + config = get_recently_viewed_config() + config.import_config() + assert config.successful_import + + test_session.expire_all() + fgs = {i.primary_path.name: i for i in test_session.query(FileGroup)} + assert set(fgs) == {'movie.mp4', 'book.epub', 'newer.mp4'} + # The FileGroup created by the import has all its files. + assert len(fgs['movie.mp4'].files) == 2 + + assert fgs['movie.mp4'].progress == 0.5 + assert fgs['movie.mp4'].position == dict(kind='time', seconds=600.0) + assert fgs['movie.mp4'].viewed == now - timedelta(hours=2) + assert fgs['book.epub'].progress == 1.0 + assert fgs['book.epub'].position is None + # The DB was newer. + assert fgs['newer.mp4'].progress == 0.8 + assert fgs['newer.mp4'].position == dict(kind='time', seconds=800.0) + + # The config round trips. + config.dump_config() + config.save() + assert [i['path'] for i in read_recently_viewed(test_directory)] == \ + ['videos/newer.mp4', 'videos/movie.mp4', 'books/book.epub'] + + +@pytest.mark.asyncio +async def test_recently_viewed_not_dumped_before_import(test_session, async_client, test_directory, + make_files_structure): + """A dump before the import would replace the history the import has yet to apply.""" + movie, = make_files_structure(['videos/movie.mp4']) + config_file = test_directory / 'config/recently_viewed.yaml' + config_file.parent.mkdir(exist_ok=True) + config_file.write_text(yaml.dump(dict(version=1, files=[ + dict(path='videos/movie.mp4', viewed=dates.now().isoformat(), progress=0.5, + position=dict(kind='time', seconds=600))]))) + + config = get_recently_viewed_config() + config.successful_import = False + config.dump_config() + assert read_recently_viewed(test_directory)[0]['progress'] == 0.5 + assert config.files == [] + + +@pytest.mark.asyncio +async def test_recently_viewed_follows_move(test_session, async_client, test_directory, make_files_structure): + """Progress follows a moved file, and the config is saved with the new path.""" + movie, = make_files_structure(['videos/movie.mp4']) + body = dict(file='videos/movie.mp4', progress=0.4, position=dict(kind='time', seconds=480), final=True) + await async_client.post('/api/files/progress', content=json.dumps(body)) + assert read_recently_viewed(test_directory)[0]['path'] == 'videos/movie.mp4' + + destination = test_directory / 'watched' + destination.mkdir() + file_worker.private_queue.put_nowait(FileTask(FileTaskType.move, [movie], destination=destination)) + await file_worker.process_queue() + # The next request awaits the switches the move activated. + await async_client.post('/api/files/file', content=json.dumps(dict(file='watched/movie.mp4', skip_tracking=True))) + + test_session.expire_all() + fg, = test_session.query(FileGroup).all() + assert fg.primary_path == destination / 'movie.mp4' + assert fg.position == dict(kind='time', seconds=480.0) + entry, = read_recently_viewed(test_directory) + assert entry['path'] == 'watched/movie.mp4' + assert entry['progress'] == 0.4 diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index ac4e989d..c676adb5 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -25,6 +25,7 @@ from wrolpi.common import get_media_directory, logger from wrolpi.db import get_db_session, get_db_curs from wrolpi.events import Events from wrolpi.vars import PYTEST +from wrolpi.files.progress import save_recently_viewed_config from wrolpi.files.lib import ( split_path_stem_and_suffix, _upsert_files, glob_shared_stem, group_files_by_stem, choose_primary_file, apply_indexers, @@ -1095,6 +1096,8 @@ class FileWorker(FileMoveMixin, FileReorganizeMixin): with flags.file_worker_cleanup: await apply_refresh_cleanup() save_tags_config.activate_switch() + # Moved, renamed, and deleted files change the recently viewed paths. + save_recently_viewed_config.activate_switch() # Each Sanic worker constructs a FileWorker. The perpetual signal is started diff --git a/wrolpi/test/test_cross_config_disaster_recovery.py b/wrolpi/test/test_cross_config_disaster_recovery.py index 880866cf..b650f3c0 100644 --- a/wrolpi/test/test_cross_config_disaster_recovery.py +++ b/wrolpi/test/test_cross_config_disaster_recovery.py @@ -164,6 +164,7 @@ class TestRealisticFullRecovery: assert 'channels' in results assert 'domains' in results assert 'inventories' in results + assert results['recently_viewed'] is True @pytest.mark.asyncio