mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Track how far the user is through a file, kept in recently_viewed.yaml
FileGroup gains `progress` (0.0 to 1.0, 1.0 is finished) and `position` (where to resume: a time, a page, or an EPUB CFI). Both are returned with the file, including by POST /api/files/file. POST /api/files/progress records progress and is also a view. A file 95% through is finished and keeps no position; one barely started (under 10 s, the first page, under 1%) is not worth resuming. The write happens before the response, so the next request sees it. POST /api/files/progress/clear starts a file over. Both refuse files outside the media directory. recently_viewed.yaml keeps the 1000 most recently viewed files so viewing history and positions survive a database rebuild. On import the newer `viewed` wins, so a file viewed while WROLPi was starting keeps its newer position. The config is saved when a file is opened, when progress is final (paused or closed), when it is cleared, and after refresh cleanup (moves, renames, deletes); heartbeats during playback only write the database. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland
parent
531721c157
commit
c8201700a8
11 changed files with 685 additions and 1 deletions
24
alembic/versions/2026_10_08_1200_file_group_progress.py
Normal file
24
alembic/versions/2026_10_08_1200_file_group_progress.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
"""A FileGroup records how far the user is through it: file_group.progress and file_group.position.
|
||||
|
||||
Revision ID: 2026_10_08_1200
|
||||
Revises: 2026_10_06_1500
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '2026_10_08_1200'
|
||||
down_revision = '2026_10_06_1500'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column('file_group', sa.Column('progress', sa.Float(), nullable=True))
|
||||
op.add_column('file_group', sa.Column('position', sa.JSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
with op.batch_alter_table('file_group') as batch_op:
|
||||
batch_op.drop_column('position')
|
||||
batch_op.drop_column('progress')
|
||||
|
|
@ -1190,6 +1190,10 @@ def get_all_configs() -> Dict[str, ConfigFile]:
|
|||
if repos_config := get_repos_config():
|
||||
all_configs[repos_config.file_name] = repos_config
|
||||
|
||||
from wrolpi.files.progress import get_recently_viewed_config
|
||||
if recently_viewed_config := get_recently_viewed_config():
|
||||
all_configs[recently_viewed_config.file_name] = recently_viewed_config
|
||||
|
||||
return all_configs
|
||||
|
||||
|
||||
|
|
@ -1316,6 +1320,16 @@ async def import_all_db_configs() -> dict[str, bool]:
|
|||
logger.warning(f'Failed to import flasher config: {e}')
|
||||
results['flasher'] = False
|
||||
|
||||
# Recently viewed files (positions in FileGroups, no dependencies)
|
||||
try:
|
||||
from wrolpi.files.progress import get_recently_viewed_config
|
||||
await asyncio.to_thread(get_recently_viewed_config().import_config)
|
||||
results['recently_viewed'] = True
|
||||
logger.debug('recently viewed config imported')
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to import recently viewed config: {e}')
|
||||
results['recently_viewed'] = False
|
||||
|
||||
return results
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ def attach_shared_contexts(app: Sanic):
|
|||
app.shared_ctx.map_pins_config = manager.dict()
|
||||
app.shared_ctx.flasher_config = manager.dict()
|
||||
app.shared_ctx.bookmarks_config = manager.dict()
|
||||
app.shared_ctx.recently_viewed_config = manager.dict()
|
||||
# Shared dicts.
|
||||
app.shared_ctx.uploaded_files = manager.dict()
|
||||
app.shared_ctx.status = manager.dict()
|
||||
|
|
@ -332,3 +333,9 @@ def initialize_configs_contexts(app: Sanic):
|
|||
BOOKMARKS_CONFIG.initialize(app.shared_ctx.bookmarks_config)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to initialize in-memory bookmarks config: {e}')
|
||||
|
||||
try:
|
||||
from wrolpi.files.progress import RECENTLY_VIEWED_CONFIG
|
||||
RECENTLY_VIEWED_CONFIG.initialize(app.shared_ctx.recently_viewed_config)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to initialize in-memory recently viewed config: {e}')
|
||||
|
|
|
|||
|
|
@ -57,6 +57,29 @@ async def get_file(request: Request, body: schema.FileRequest):
|
|||
return json_response({'file': file})
|
||||
|
||||
|
||||
@files_bp.post('/progress')
|
||||
@openapi.definition(
|
||||
summary='Record how far the user is through a file, and where to resume it',
|
||||
body=schema.FileProgressRequest,
|
||||
)
|
||||
@validate(schema.FileProgressRequest)
|
||||
async def post_file_progress(request: Request, body: schema.FileProgressRequest):
|
||||
path = get_media_directory() / body.file
|
||||
progress, position = lib.set_file_progress(request.ctx.session, path, body.progress, body.position, body.final)
|
||||
return json_response({'progress': progress, 'position': position})
|
||||
|
||||
|
||||
@files_bp.post('/progress/clear')
|
||||
@openapi.definition(
|
||||
summary='Forget how far the user is through a file, so it starts from the beginning',
|
||||
body=schema.FileProgressClearRequest,
|
||||
)
|
||||
@validate(schema.FileProgressClearRequest)
|
||||
async def post_file_progress_clear(request: Request, body: schema.FileProgressClearRequest):
|
||||
lib.clear_file_progress(request.ctx.session, get_media_directory() / body.file)
|
||||
return response.empty(HTTPStatus.NO_CONTENT)
|
||||
|
||||
|
||||
@files_bp.post('/delete')
|
||||
@openapi.definition(
|
||||
summary='Delete files or directories. Directories are deleted recursively.'
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
|
|||
NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError, UnsupportedArchive, InvalidArchiveMember
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.models import FileGroup, Directory
|
||||
from wrolpi.files.progress import normalize_progress, save_recently_viewed_config
|
||||
from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES
|
||||
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config, sync_tags_directory
|
||||
from wrolpi.vars import PYTEST, IS_MACOS
|
||||
|
|
@ -124,11 +125,14 @@ def _get_file_dict(session: Session, file: pathlib.Path) -> Dict:
|
|||
mimetype = get_mimetype(file)
|
||||
except PermissionError:
|
||||
mimetype = None
|
||||
file_group = FileGroup.get_by_path(session, file)
|
||||
return dict(
|
||||
path=file.relative_to(media_directory),
|
||||
size=size,
|
||||
mimetype=mimetype,
|
||||
tags=get_file_tag_names(session, file),
|
||||
progress=file_group.progress if file_group else None,
|
||||
position=file_group.position if file_group else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -156,6 +160,53 @@ async def set_file_viewed(session: Session, file: pathlib.Path):
|
|||
session.commit()
|
||||
|
||||
|
||||
def _get_or_create_file_group(session: Session, file: pathlib.Path) -> FileGroup:
|
||||
if not get_paths_in_media_directory([file]):
|
||||
raise InvalidFile('File must be in the media directory')
|
||||
try:
|
||||
return FileGroup.find_by_path(session, file)
|
||||
except UnknownFile:
|
||||
if not file.is_file():
|
||||
raise InvalidFile(f'File does not exist: {get_relative_to_media_directory(file)}')
|
||||
fg = FileGroup.from_paths(session, file)
|
||||
fg.do_model(session)
|
||||
return fg
|
||||
|
||||
|
||||
def set_file_progress(session: Session, file: pathlib.Path, progress: float, position: dict = None,
|
||||
final: bool = False) -> Tuple[float | None, dict | None]:
|
||||
"""Record how far the User is through a file (see wrolpi.files.progress). This is also a view.
|
||||
|
||||
@param final: The User stopped (paused, closed); save the recently viewed config.
|
||||
@return: The (progress, position) that was stored.
|
||||
"""
|
||||
progress, position = normalize_progress(progress, position)
|
||||
|
||||
# Files in ignored directories are not tracked.
|
||||
if not remove_files_in_ignored_directories([file]):
|
||||
return None, None
|
||||
|
||||
fg = _get_or_create_file_group(session, file)
|
||||
# Not `set_viewed`, which saves the config; a heartbeat during playback only writes the DB.
|
||||
fg.viewed = now()
|
||||
fg.progress = progress
|
||||
fg.position = position
|
||||
session.commit()
|
||||
|
||||
if final:
|
||||
save_recently_viewed_config.activate_switch()
|
||||
return progress, position
|
||||
|
||||
|
||||
def clear_file_progress(session: Session, file: pathlib.Path):
|
||||
"""Forget how far the User is through a file, so it starts from the beginning. It remains viewed."""
|
||||
fg = _get_or_create_file_group(session, file)
|
||||
fg.progress = None
|
||||
fg.position = None
|
||||
session.commit()
|
||||
save_recently_viewed_config.activate_switch()
|
||||
|
||||
|
||||
@cachetools.func.ttl_cache(10_000, 30.0)
|
||||
def _get_directory_dict(directory: pathlib.Path,
|
||||
directories_cache: str, # Used to cache by requested directories.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import urllib.parse
|
|||
from datetime import datetime
|
||||
from typing import List, Type, Optional, Iterable
|
||||
|
||||
from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON, or_
|
||||
from sqlalchemy import Column, String, BigInteger, Boolean, event, Float, Index, Integer, JSON, or_
|
||||
from sqlalchemy import types
|
||||
from sqlalchemy.orm import deferred, relationship, Session, selectinload
|
||||
|
||||
|
|
@ -18,6 +18,7 @@ from wrolpi.downloader import Download
|
|||
from wrolpi.errors import FileGroupIsTagged, UnknownFile
|
||||
from wrolpi.files import indexers
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.files.progress import save_recently_viewed_config
|
||||
from wrolpi.tags import Tag, TagFile, save_tags_config, sync_tags_directory
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
|
|
@ -116,6 +117,9 @@ class FileGroup(ModelHelper, Base):
|
|||
title = Column(String) # user-displayable title
|
||||
url = Column(String) # the location where this file can be downloaded.
|
||||
viewed = Column(TZDateTime) # the most recent time a User viewed this file.
|
||||
# How far the User is through this file, see wrolpi.files.progress.
|
||||
progress = Column(Float) # 0.0 to 1.0, where 1.0 is finished.
|
||||
position = Column(JSON) # Where to resume, e.g. {"kind": "time", "seconds": 512.3}.
|
||||
|
||||
# Columns updated by triggers.
|
||||
# `file_group_effective_datetime_trigger` and `update_effective_datetime` event handler
|
||||
|
|
@ -180,6 +184,8 @@ class FileGroup(ModelHelper, Base):
|
|||
title=self.title,
|
||||
url=self.url,
|
||||
viewed=self.viewed,
|
||||
progress=self.progress,
|
||||
position=self.position,
|
||||
)
|
||||
return d
|
||||
|
||||
|
|
@ -224,6 +230,7 @@ class FileGroup(ModelHelper, Base):
|
|||
:return: The datetime that was set.
|
||||
"""
|
||||
self.viewed = viewed or now()
|
||||
save_recently_viewed_config.activate_switch()
|
||||
return self.viewed
|
||||
|
||||
def set_tags(self, session: Session, tag_names_or_ids: Iterable[str | int]):
|
||||
|
|
|
|||
238
wrolpi/files/progress.py
Normal file
238
wrolpi/files/progress.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
"""How far the User is through a file, and the recently viewed files config.
|
||||
|
||||
`FileGroup.progress` is the fraction consumed (0.0 to 1.0, where 1.0 is finished), and `FileGroup.position`
|
||||
is where to resume. A position is a dict with a `kind`:
|
||||
|
||||
{"kind": "time", "seconds": 512.3} video and audio
|
||||
{"kind": "epub", "cfi": "epubcfi(/6/14!/4/2)"} EPUB
|
||||
{"kind": "page", "page": 12} comic books (0-based)
|
||||
|
||||
The client may add `updated_at` (milliseconds since the epoch) so it can compare the server's position with
|
||||
one it holds locally.
|
||||
|
||||
The database is the working store. `recently_viewed.yaml` holds the most recently viewed files so viewing
|
||||
history and positions survive a database rebuild.
|
||||
"""
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import pytz
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.common import ConfigFile, logger, get_media_directory
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import ValidationError, NoPrimaryFile
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
# A file this far through is finished; the rest is credits, an end screen, or an index.
|
||||
FINISHED_PROGRESS = 0.95
|
||||
# A position this close to the start is not worth resuming.
|
||||
MINIMUM_SECONDS = 10
|
||||
MINIMUM_PROGRESS = 0.01
|
||||
MAXIMUM_CFI_LENGTH = 1_000
|
||||
|
||||
# The most recently viewed files kept in the config. The database keeps every `viewed`.
|
||||
RECENTLY_VIEWED_LIMIT = 1_000
|
||||
|
||||
|
||||
def _number(value, name: str) -> float:
|
||||
# bool is an int; reject it so `true` is not a position.
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
||||
raise ValidationError(f'{name} must be a number')
|
||||
return value
|
||||
|
||||
|
||||
def validate_position(position: Optional[dict]) -> Optional[dict]:
|
||||
"""Return a clean copy of `position`, or raise ValidationError."""
|
||||
if position is None:
|
||||
return None
|
||||
if not isinstance(position, dict):
|
||||
raise ValidationError('position must be an object')
|
||||
|
||||
kind = position.get('kind')
|
||||
if kind == 'time':
|
||||
clean = dict(kind=kind, seconds=float(_number(position.get('seconds'), 'seconds')))
|
||||
if clean['seconds'] < 0:
|
||||
raise ValidationError('seconds cannot be negative')
|
||||
elif kind == 'page':
|
||||
page = position.get('page')
|
||||
if isinstance(page, bool) or not isinstance(page, int) or page < 0:
|
||||
raise ValidationError('page must be a non-negative integer')
|
||||
clean = dict(kind=kind, page=page)
|
||||
elif kind == 'epub':
|
||||
cfi = position.get('cfi')
|
||||
if not isinstance(cfi, str) or not cfi.startswith('epubcfi(') or len(cfi) > MAXIMUM_CFI_LENGTH:
|
||||
raise ValidationError('cfi must be an epubcfi')
|
||||
clean = dict(kind=kind, cfi=cfi)
|
||||
else:
|
||||
raise ValidationError(f'Unknown position kind: {kind!r}')
|
||||
|
||||
if (updated_at := position.get('updated_at')) is not None:
|
||||
clean['updated_at'] = int(_number(updated_at, 'updated_at'))
|
||||
return clean
|
||||
|
||||
|
||||
def normalize_progress(progress, position: Optional[dict]) -> Tuple[float, Optional[dict]]:
|
||||
"""Apply the progress rules: a finished file has no position, nor does one barely started.
|
||||
|
||||
@return: (progress, position) to store.
|
||||
"""
|
||||
progress = min(max(float(_number(progress, 'progress')), 0.0), 1.0)
|
||||
position = validate_position(position)
|
||||
|
||||
if progress >= FINISHED_PROGRESS:
|
||||
return 1.0, None
|
||||
|
||||
too_early = progress < MINIMUM_PROGRESS
|
||||
if position and position['kind'] == 'time':
|
||||
too_early = too_early or position['seconds'] < MINIMUM_SECONDS
|
||||
elif position and position['kind'] == 'page':
|
||||
too_early = too_early or position['page'] == 0
|
||||
if too_early:
|
||||
return 0.0, None
|
||||
|
||||
return progress, position
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecentlyViewedConfigValidator:
|
||||
version: int = None
|
||||
files: list = field(default_factory=list)
|
||||
|
||||
|
||||
def _parse_entry(entry) -> Optional[Tuple[str, datetime, Optional[float], Optional[dict]]]:
|
||||
"""Return (path, viewed, progress, position) from a config entry, or None if it is malformed."""
|
||||
if not isinstance(entry, dict) or not isinstance(entry.get('path'), str) or not entry.get('viewed'):
|
||||
return None
|
||||
try:
|
||||
viewed = entry['viewed']
|
||||
if isinstance(viewed, datetime):
|
||||
# YAML parses an unquoted timestamp (a hand edit) into a naive UTC datetime.
|
||||
viewed = viewed if viewed.tzinfo else viewed.replace(tzinfo=pytz.UTC)
|
||||
else:
|
||||
viewed = dates.strptime_ms(viewed)
|
||||
progress, position = None, None
|
||||
if entry.get('progress') is not None:
|
||||
progress, position = normalize_progress(entry['progress'], entry.get('position'))
|
||||
return entry['path'], viewed, progress, position
|
||||
except Exception as e:
|
||||
logger.warning(f'Ignoring invalid recently viewed entry {entry!r}: {e}')
|
||||
return None
|
||||
|
||||
|
||||
class RecentlyViewedConfig(ConfigFile):
|
||||
file_name = 'recently_viewed.yaml'
|
||||
default_config = dict(
|
||||
version=0,
|
||||
files=[],
|
||||
)
|
||||
validator = RecentlyViewedConfigValidator
|
||||
width = 500
|
||||
|
||||
@property
|
||||
def files(self) -> List[dict]:
|
||||
return list(self._config.get('files', []))
|
||||
|
||||
def dump_config(self, file=None, send_events=False, overwrite=False):
|
||||
if self.get_file().exists() and not self.successful_import and not overwrite:
|
||||
# Dumping now would replace history the import has not yet applied.
|
||||
logger.debug(f'Not dumping {self.file_name} because it has not been imported')
|
||||
return
|
||||
|
||||
try:
|
||||
# wrolpi.files.models imports this module.
|
||||
from wrolpi.files.models import FileGroup
|
||||
media_directory = get_media_directory()
|
||||
with get_db_session() as session:
|
||||
file_groups = session.query(FileGroup) \
|
||||
.filter(FileGroup.viewed.isnot(None)) \
|
||||
.order_by(FileGroup.viewed.desc(), FileGroup.id) \
|
||||
.limit(RECENTLY_VIEWED_LIMIT)
|
||||
files = list()
|
||||
for file_group in file_groups:
|
||||
entry = dict(
|
||||
path=str(file_group.primary_path.relative_to(media_directory)),
|
||||
viewed=file_group.viewed.isoformat(),
|
||||
)
|
||||
if file_group.progress is not None:
|
||||
entry['progress'] = file_group.progress
|
||||
if file_group.position:
|
||||
entry['position'] = file_group.position
|
||||
files.append(entry)
|
||||
self.update({'files': files}, overwrite=overwrite)
|
||||
except Exception as e:
|
||||
message = f'Failed to save {self.get_relative_file()} config'
|
||||
logger.error(message, exc_info=e)
|
||||
if send_events:
|
||||
Events.send_config_save_failed(message)
|
||||
|
||||
def import_config(self, file=None, send_events=False):
|
||||
"""Apply the config to the database. The newer `viewed` wins, so a file viewed while WROLPi was
|
||||
starting keeps its newer position."""
|
||||
super().import_config(file, send_events)
|
||||
try:
|
||||
# wrolpi.files.models imports this module.
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
media_directory = get_media_directory()
|
||||
entries = [i for i in map(_parse_entry, self.files) if i]
|
||||
with get_db_session(commit=True) as session:
|
||||
absolute_paths = [media_directory / path for path, *_ in entries]
|
||||
file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(absolute_paths))
|
||||
file_groups_by_path = {i.primary_path: i for i in file_groups}
|
||||
|
||||
applied = 0
|
||||
for path, viewed, progress, position in entries:
|
||||
absolute_path = media_directory / path
|
||||
file_group = file_groups_by_path.get(absolute_path)
|
||||
if not file_group:
|
||||
if not absolute_path.is_file():
|
||||
# The file was deleted; its history goes with it.
|
||||
logger.debug(f'Not importing recently viewed file which does not exist: {path}')
|
||||
continue
|
||||
try:
|
||||
file_group = FileGroup.from_paths(session, *glob_shared_stem(absolute_path))
|
||||
except NoPrimaryFile:
|
||||
logger.warning(f'Cannot import recently viewed file: {path}')
|
||||
continue
|
||||
session.add(file_group)
|
||||
session.flush([file_group])
|
||||
file_groups_by_path[absolute_path] = file_group
|
||||
|
||||
if file_group.viewed and file_group.viewed >= viewed:
|
||||
continue
|
||||
file_group.viewed = viewed
|
||||
file_group.progress = progress
|
||||
file_group.position = position
|
||||
applied += 1
|
||||
|
||||
logger.info(f'Imported {applied} of {len(entries)} recently viewed files')
|
||||
self.successful_import = True
|
||||
except Exception as e:
|
||||
self.successful_import = False
|
||||
message = f'Failed to import {self.get_relative_file()} config'
|
||||
logger.error(message, exc_info=e)
|
||||
if send_events:
|
||||
Events.send_config_import_failed(message)
|
||||
raise
|
||||
|
||||
|
||||
RECENTLY_VIEWED_CONFIG: RecentlyViewedConfig = RecentlyViewedConfig()
|
||||
|
||||
|
||||
def get_recently_viewed_config() -> RecentlyViewedConfig:
|
||||
return RECENTLY_VIEWED_CONFIG
|
||||
|
||||
|
||||
@register_switch_handler('save_recently_viewed_config')
|
||||
def save_recently_viewed_config():
|
||||
get_recently_viewed_config().dump_config()
|
||||
|
||||
|
||||
save_recently_viewed_config: ActivateSwitchMethod
|
||||
|
|
@ -15,6 +15,19 @@ class FileRequest:
|
|||
skip_tracking: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileProgressRequest:
|
||||
file: str
|
||||
progress: float
|
||||
position: Optional[dict] = None
|
||||
final: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileProgressClearRequest:
|
||||
file: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeleteRequest:
|
||||
paths: List[str] = field(default_factory=list)
|
||||
|
|
|
|||
303
wrolpi/files/test/test_progress.py
Normal file
303
wrolpi/files/test/test_progress.py
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
"""Tests for file progress (how far the user is through a file) and recently_viewed.yaml."""
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.progress import normalize_progress, get_recently_viewed_config, RECENTLY_VIEWED_LIMIT
|
||||
from wrolpi.files.worker import file_worker, FileTask, FileTaskType
|
||||
|
||||
|
||||
def read_recently_viewed(test_directory) -> list:
|
||||
with (test_directory / 'config/recently_viewed.yaml').open() as fh:
|
||||
return yaml.safe_load(fh)['files']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('progress,position,expected', [
|
||||
# A video part way through keeps its position.
|
||||
(0.5, dict(kind='time', seconds=300.0), (0.5, dict(kind='time', seconds=300.0))),
|
||||
# Finished files have no position.
|
||||
(0.95, dict(kind='time', seconds=570), (1.0, None)),
|
||||
(1.5, dict(kind='page', page=40), (1.0, None)),
|
||||
# Barely started files are not worth resuming.
|
||||
(0.2, dict(kind='time', seconds=9.9), (0.0, None)),
|
||||
(0.005, dict(kind='epub', cfi='epubcfi(/6/2!/4/2)'), (0.0, None)),
|
||||
(0.1, dict(kind='page', page=0), (0.0, None)),
|
||||
(-1, None, (0.0, None)),
|
||||
# Pages and CFIs.
|
||||
(0.3, dict(kind='page', page=12), (0.3, dict(kind='page', page=12))),
|
||||
(0.3, dict(kind='epub', cfi='epubcfi(/6/14!/4/2)', updated_at=1760000000000),
|
||||
(0.3, dict(kind='epub', cfi='epubcfi(/6/14!/4/2)', updated_at=1760000000000))),
|
||||
# Unknown keys are dropped.
|
||||
(0.3, dict(kind='page', page=3, evil='x'), (0.3, dict(kind='page', page=3))),
|
||||
])
|
||||
def test_normalize_progress(progress, position, expected):
|
||||
assert normalize_progress(progress, position) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize('progress,position', [
|
||||
('half', None),
|
||||
(True, None),
|
||||
(float('nan'), None),
|
||||
(0.5, dict(kind='time', seconds='300')),
|
||||
(0.5, dict(kind='time', seconds=-1)),
|
||||
(0.5, dict(kind='page', page=1.5)),
|
||||
(0.5, dict(kind='page', page=True)),
|
||||
(0.5, dict(kind='epub', cfi='javascript:alert(1)')),
|
||||
(0.5, dict(kind='epub', cfi='epubcfi(' + 'a' * 1_000 + ')')),
|
||||
(0.5, dict(kind='scroll', fraction=0.5)),
|
||||
(0.5, 'time'),
|
||||
])
|
||||
def test_normalize_progress_invalid(progress, position):
|
||||
with pytest.raises(ValidationError):
|
||||
normalize_progress(progress, position)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_api(test_session, async_client, test_directory, make_files_structure):
|
||||
"""Progress is stored on the FileGroup, returned with the file, and is a view."""
|
||||
make_files_structure(['videos/movie.mp4'])
|
||||
|
||||
body = dict(file='videos/movie.mp4', progress=0.4, position=dict(kind='time', seconds=480.5))
|
||||
request, response = await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json == dict(progress=0.4, position=dict(kind='time', seconds=480.5))
|
||||
|
||||
fg, = test_session.query(FileGroup).all()
|
||||
assert fg.viewed
|
||||
assert fg.progress == 0.4
|
||||
assert fg.position == dict(kind='time', seconds=480.5)
|
||||
|
||||
# The next request sees what was stored.
|
||||
request, response = await async_client.post('/api/files/file', content=json.dumps(
|
||||
dict(file='videos/movie.mp4', skip_tracking=True)))
|
||||
assert response.json['file']['progress'] == 0.4
|
||||
assert response.json['file']['position'] == dict(kind='time', seconds=480.5)
|
||||
|
||||
# Rewinding is allowed; the last write wins.
|
||||
body = dict(file='videos/movie.mp4', progress=0.2, position=dict(kind='time', seconds=240))
|
||||
request, response = await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
test_session.expire_all()
|
||||
assert fg.progress == 0.2
|
||||
assert fg.position == dict(kind='time', seconds=240)
|
||||
|
||||
# Start over.
|
||||
request, response = await async_client.post('/api/files/progress/clear',
|
||||
content=json.dumps(dict(file='videos/movie.mp4')))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
test_session.expire_all()
|
||||
assert fg.progress is None
|
||||
assert fg.position is None
|
||||
assert fg.viewed, 'A file that starts over has still been viewed'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_api_errors(test_session, async_client, test_directory, make_files_structure):
|
||||
make_files_structure(['videos/movie.mp4'])
|
||||
|
||||
body = dict(file='videos/movie.mp4', progress=0.4, position=dict(kind='scroll', fraction=0.4))
|
||||
request, response = await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
body = dict(file='videos/missing.mp4', progress=0.4, position=dict(kind='time', seconds=40))
|
||||
request, response = await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
# A file outside the media directory.
|
||||
outside = test_directory.parent / 'outside.mp4'
|
||||
outside.touch()
|
||||
try:
|
||||
for file in ('../outside.mp4', str(outside)):
|
||||
body = dict(file=file, progress=0.4, position=dict(kind='time', seconds=40))
|
||||
request, response = await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
request, response = await async_client.post('/api/files/progress/clear',
|
||||
content=json.dumps(dict(file=file)))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
finally:
|
||||
outside.unlink()
|
||||
|
||||
assert test_session.query(FileGroup).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_ignored_directory(test_session, async_client, test_directory, make_files_structure,
|
||||
test_wrolpi_config, await_switches):
|
||||
"""Files in ignored directories are not tracked."""
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
make_files_structure(['private/movie.mp4'])
|
||||
get_wrolpi_config().ignored_directories = [str(test_directory / 'private')]
|
||||
await await_switches()
|
||||
|
||||
body = dict(file='private/movie.mp4', progress=0.4, position=dict(kind='time', seconds=480))
|
||||
request, response = await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert test_session.query(FileGroup).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_saves_config_when_final(test_session, async_client, test_directory, make_files_structure):
|
||||
"""Only a final progress (pause, close) saves the config; a heartbeat during playback writes only the DB."""
|
||||
make_files_structure(['videos/movie.mp4'])
|
||||
config_file = test_directory / 'config/recently_viewed.yaml'
|
||||
|
||||
body = dict(file='videos/movie.mp4', progress=0.4, position=dict(kind='time', seconds=480))
|
||||
await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
assert not config_file.exists()
|
||||
|
||||
body['final'] = True
|
||||
await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
entry, = read_recently_viewed(test_directory)
|
||||
assert entry['path'] == 'videos/movie.mp4'
|
||||
assert entry['progress'] == 0.4
|
||||
assert entry['position'] == dict(kind='time', seconds=480.0)
|
||||
assert dates.strptime_ms(entry['viewed'])
|
||||
|
||||
# Starting over saves the config.
|
||||
await async_client.post('/api/files/progress/clear', content=json.dumps(dict(file='videos/movie.mp4')))
|
||||
entry, = read_recently_viewed(test_directory)
|
||||
assert 'progress' not in entry and 'position' not in entry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_view_saves_config(test_session, async_client, test_directory, make_files_structure, refresh_files,
|
||||
await_background_tasks, await_switches):
|
||||
"""Opening a file saves it in the recently viewed config."""
|
||||
make_files_structure({'docs/notes.txt': 'notes'})
|
||||
await refresh_files()
|
||||
await await_switches()
|
||||
# The refresh saved the config, with nothing viewed yet.
|
||||
assert read_recently_viewed(test_directory) == []
|
||||
|
||||
await async_client.post('/api/files/file', content=json.dumps(dict(file='docs/notes.txt')))
|
||||
# Viewing a file is a background task, which activates the switch.
|
||||
await await_background_tasks()
|
||||
await await_switches()
|
||||
|
||||
entry, = read_recently_viewed(test_directory)
|
||||
assert entry['path'] == 'docs/notes.txt'
|
||||
assert 'progress' not in entry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recently_viewed_dump_order_and_limit(test_session, async_client, test_directory, make_files_structure,
|
||||
monkeypatch):
|
||||
"""The config holds the most recently viewed files, newest first."""
|
||||
monkeypatch.setattr('wrolpi.files.progress.RECENTLY_VIEWED_LIMIT', 2)
|
||||
old, middle, new, never = make_files_structure(['old.txt', 'middle.txt', 'new.txt', 'never.txt'])
|
||||
now = dates.now()
|
||||
for path, viewed in ((old, now - timedelta(days=2)), (middle, now - timedelta(days=1)), (new, now), (never, None)):
|
||||
fg = FileGroup.from_paths(test_session, path)
|
||||
fg.viewed = viewed
|
||||
test_session.commit()
|
||||
|
||||
config = get_recently_viewed_config()
|
||||
config.dump_config()
|
||||
config.save()
|
||||
|
||||
assert [i['path'] for i in read_recently_viewed(test_directory)] == ['new.txt', 'middle.txt']
|
||||
assert RECENTLY_VIEWED_LIMIT == 1_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recently_viewed_import(test_session, async_client, test_directory, make_files_structure,
|
||||
video_file_factory):
|
||||
"""A wiped database gets its viewing history back from the config. The newer `viewed` wins."""
|
||||
_, book, newer = make_files_structure(['videos/movie.en.vtt', 'books/book.epub', 'videos/newer.mp4'])
|
||||
video_file_factory(test_directory / 'videos/movie.mp4')
|
||||
now = dates.now()
|
||||
|
||||
# This file was viewed after the config was written (e.g. while WROLPi was starting).
|
||||
newer_fg = FileGroup.from_paths(test_session, newer)
|
||||
newer_fg.viewed = now
|
||||
newer_fg.progress = 0.8
|
||||
newer_fg.position = dict(kind='time', seconds=800.0)
|
||||
test_session.commit()
|
||||
|
||||
config_file = test_directory / 'config/recently_viewed.yaml'
|
||||
config_file.parent.mkdir(exist_ok=True)
|
||||
config_file.write_text(yaml.dump(dict(version=3, files=[
|
||||
dict(path='videos/newer.mp4', viewed=(now - timedelta(hours=1)).isoformat(), progress=0.1,
|
||||
position=dict(kind='time', seconds=100)),
|
||||
dict(path='videos/movie.mp4', viewed=(now - timedelta(hours=2)).isoformat(), progress=0.5,
|
||||
position=dict(kind='time', seconds=600)),
|
||||
dict(path='books/book.epub', viewed=(now - timedelta(hours=3)).isoformat(), progress=1.0),
|
||||
# The file was deleted.
|
||||
dict(path='videos/deleted.mp4', viewed=(now - timedelta(hours=4)).isoformat(), progress=0.5),
|
||||
# Invalid entries are ignored.
|
||||
dict(path='videos/movie.mp4', viewed='not a date'),
|
||||
dict(path='books/book.epub', viewed=now.isoformat(), progress=0.5, position=dict(kind='scroll')),
|
||||
'not an entry',
|
||||
])))
|
||||
|
||||
config = get_recently_viewed_config()
|
||||
config.import_config()
|
||||
assert config.successful_import
|
||||
|
||||
test_session.expire_all()
|
||||
fgs = {i.primary_path.name: i for i in test_session.query(FileGroup)}
|
||||
assert set(fgs) == {'movie.mp4', 'book.epub', 'newer.mp4'}
|
||||
# The FileGroup created by the import has all its files.
|
||||
assert len(fgs['movie.mp4'].files) == 2
|
||||
|
||||
assert fgs['movie.mp4'].progress == 0.5
|
||||
assert fgs['movie.mp4'].position == dict(kind='time', seconds=600.0)
|
||||
assert fgs['movie.mp4'].viewed == now - timedelta(hours=2)
|
||||
assert fgs['book.epub'].progress == 1.0
|
||||
assert fgs['book.epub'].position is None
|
||||
# The DB was newer.
|
||||
assert fgs['newer.mp4'].progress == 0.8
|
||||
assert fgs['newer.mp4'].position == dict(kind='time', seconds=800.0)
|
||||
|
||||
# The config round trips.
|
||||
config.dump_config()
|
||||
config.save()
|
||||
assert [i['path'] for i in read_recently_viewed(test_directory)] == \
|
||||
['videos/newer.mp4', 'videos/movie.mp4', 'books/book.epub']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recently_viewed_not_dumped_before_import(test_session, async_client, test_directory,
|
||||
make_files_structure):
|
||||
"""A dump before the import would replace the history the import has yet to apply."""
|
||||
movie, = make_files_structure(['videos/movie.mp4'])
|
||||
config_file = test_directory / 'config/recently_viewed.yaml'
|
||||
config_file.parent.mkdir(exist_ok=True)
|
||||
config_file.write_text(yaml.dump(dict(version=1, files=[
|
||||
dict(path='videos/movie.mp4', viewed=dates.now().isoformat(), progress=0.5,
|
||||
position=dict(kind='time', seconds=600))])))
|
||||
|
||||
config = get_recently_viewed_config()
|
||||
config.successful_import = False
|
||||
config.dump_config()
|
||||
assert read_recently_viewed(test_directory)[0]['progress'] == 0.5
|
||||
assert config.files == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recently_viewed_follows_move(test_session, async_client, test_directory, make_files_structure):
|
||||
"""Progress follows a moved file, and the config is saved with the new path."""
|
||||
movie, = make_files_structure(['videos/movie.mp4'])
|
||||
body = dict(file='videos/movie.mp4', progress=0.4, position=dict(kind='time', seconds=480), final=True)
|
||||
await async_client.post('/api/files/progress', content=json.dumps(body))
|
||||
assert read_recently_viewed(test_directory)[0]['path'] == 'videos/movie.mp4'
|
||||
|
||||
destination = test_directory / 'watched'
|
||||
destination.mkdir()
|
||||
file_worker.private_queue.put_nowait(FileTask(FileTaskType.move, [movie], destination=destination))
|
||||
await file_worker.process_queue()
|
||||
# The next request awaits the switches the move activated.
|
||||
await async_client.post('/api/files/file', content=json.dumps(dict(file='watched/movie.mp4', skip_tracking=True)))
|
||||
|
||||
test_session.expire_all()
|
||||
fg, = test_session.query(FileGroup).all()
|
||||
assert fg.primary_path == destination / 'movie.mp4'
|
||||
assert fg.position == dict(kind='time', seconds=480.0)
|
||||
entry, = read_recently_viewed(test_directory)
|
||||
assert entry['path'] == 'watched/movie.mp4'
|
||||
assert entry['progress'] == 0.4
|
||||
|
|
@ -25,6 +25,7 @@ from wrolpi.common import get_media_directory, logger
|
|||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.vars import PYTEST
|
||||
from wrolpi.files.progress import save_recently_viewed_config
|
||||
from wrolpi.files.lib import (
|
||||
split_path_stem_and_suffix, _upsert_files, glob_shared_stem,
|
||||
group_files_by_stem, choose_primary_file, apply_indexers,
|
||||
|
|
@ -1095,6 +1096,8 @@ class FileWorker(FileMoveMixin, FileReorganizeMixin):
|
|||
with flags.file_worker_cleanup:
|
||||
await apply_refresh_cleanup()
|
||||
save_tags_config.activate_switch()
|
||||
# Moved, renamed, and deleted files change the recently viewed paths.
|
||||
save_recently_viewed_config.activate_switch()
|
||||
|
||||
|
||||
# Each Sanic worker constructs a FileWorker. The perpetual signal is started
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ class TestRealisticFullRecovery:
|
|||
assert 'channels' in results
|
||||
assert 'domains' in results
|
||||
assert 'inventories' in results
|
||||
assert results['recently_viewed'] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue