rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

playlists.yaml imports by diffing (#869)

Browse source 
* playlists.yaml imports by diffing

playlists.yaml declares two sections: playlists, keyed by name, and
their items, keyed by what each points to (a file's or Zim's
media-relative path and entry, or a URL) with position and title
compared.  A playlist has at most one item per reference, so a moved
item changes only its position and keeps its row; an unchanged import
writes nothing (every import deleted and re-created every item).

prepare skips items whose file or Zim is not indexed before the write
(their place is kept), drops a Tag that no longer exists once tags.yaml
is imported (the config is then saved), and, without deletes, keeps the
database's order and adds the file's other items after the item that
precedes them.

Moving a FileGroup in a playlist counts as a playlists.yaml change (ORM,
the bulk move, and a refresh that changes the primary file).

media_relative moves to wrolpi.config_import for both configs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Playlist items: keep database order without deletes; empty lists delete

Without a database version, every playlist with an added or changed item
keeps the database's order (the file's new items go after their
predecessor; only titles come from the file), and a row whose merged
position and title equal the database's is not written.  Before, only
playlists that gained an item were merged, so a file that only reordered
(or lacked an item the database had) gave two items one position.

A section can say it is present without rows (`ConfigSection.present`):
a listed playlist with an empty items list has its items removed; an
empty or missing playlists list still removes nothing.  An item that
cannot be read, or repeats another, is counted in import_skipped, so the
config is not saved over it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-25 09:42:12 -06:00 • committed by Roland
parent b67d51b69a
commit adbcf944a5
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
8 changed files with 596 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

402
wrolpi/collections/config.py
View file

@ -1,12 +1,15 @@
import pathlib
import shutil
from dataclasses import dataclass, field
from typing import List
from typing import Dict, List, Optional, Set, Tuple
from sqlalchemy import insert
from sqlalchemy.orm import Session
from wrolpi.common import ConfigFile, get_media_directory, get_relative_to_media_directory, logger
from wrolpi.config_version import IMPORT, SAVE, count_config_change
from wrolpi.config_import import ConfigSection, SectionDiff, chunks, media_relative
from wrolpi.config_version import count_config_change
from wrolpi.db import get_db_session
from wrolpi.events import Events
from wrolpi.switches import ActivateSwitchMethod, register_switch_handler
from wrolpi.tags import TagFile, TagsConfig, save_tags_config
@ -15,9 +18,195 @@ logger = logger.getChild(__name__)
__all__ = ['PlaylistsConfig', 'playlists_config', 'save_playlists_config', 'get_playlists_config']
def _item_key(item) -> tuple:
"""What a playlist item points to; a playlist has at most one item per key (see CollectionItem's constraints)."""
return item.item_kind, item.file_group_id, item.zim_id, item.zim_entry, item.url
def _playlist_tag_names(rows) -> Set[str]:
return {i['tag_name'] for i in rows if i['tag_name']}
class PlaylistsSection(ConfigSection):
"""playlists.yaml's playlists (Collections with kind 'playlist'), keyed by name."""
name = 'playlists'
def file_rows(self, config: dict) -> Tuple[Dict[str, dict], Set[str]]:
rows, invalid = dict(), set()
for data in config.get('playlists') or []:
name = data.get('name')
if not name:
logger.error(f'Playlist config entry has no name, skipping: {data}')
continue
try:
rows[name] = dict(
description=data.get('description') or None,
# Optional custom directory (media-relative); None when not customized.
directory=media_relative(data['directory']) if data.get('directory') else None,
tag_name=data.get('tag_name') or None,
)
except Exception as e:
logger.error(f'Skipping invalid playlist in config: {name}', exc_info=e)
invalid.add(name)
return rows, invalid
def db_rows(self, session: Session) -> Dict[str, dict]:
from wrolpi.tags import Tag
from .models import Collection
query = session.query(Collection.name, Collection.description, Collection.directory, Tag.name) \
.outerjoin(Tag, Tag.id == Collection.tag_id) \
.filter(Collection.kind == 'playlist')
return {name: dict(description=description or None,
directory=media_relative(directory) if directory else None,
tag_name=tag_name)
for name, description, directory, tag_name in query}
def upsert(self, session: Session, rows: Dict[str, dict]):
from wrolpi.tags import Tag
from .models import Collection
existing = {i.name: i for i in
session.query(Collection).filter(Collection.kind == 'playlist', Collection.name.in_(list(rows)))}
tags = {i.name: i for i in session.query(Tag).filter(Tag.name.in_(list(_playlist_tag_names(rows.values()))))}
for name, row in rows.items():
if not (collection := existing.get(name)):
collection = Collection(name=name, kind='playlist')
session.add(collection)
collection.description = row['description']
collection.directory = (get_media_directory() / row['directory']) if row['directory'] else None
collection.tag = tags.get(row['tag_name'])
session.flush()
def delete(self, session: Session, keys: Set[str]):
from .models import Collection
for collection in session.query(Collection).filter(Collection.kind == 'playlist',
Collection.name.in_(list(keys))):
logger.info(f'Deleting playlist {collection.name!r} (no longer in config)')
session.delete(collection)
class PlaylistItemsSection(ConfigSection):
"""playlists.yaml's playlist items, keyed by (playlist name, kind, reference, Zim entry): a file's or Zim's
media-relative path, or a URL. A playlist has at most one item per reference (CollectionItem's constraints), so
moving an item changes only its position, and it keeps its row."""
name = 'items'
@staticmethod
def item_key(name: str, data: dict) -> Optional[tuple]:
if 'file' in data:
return name, 'file', media_relative(data['file']), None
elif 'zim' in data:
if not data.get('entry'):
raise ValueError('A zim item requires an entry')
return name, 'zim', media_relative(data['zim']), data['entry']
elif 'url' in data:
return name, 'url', data['url'], None
raise ValueError('An item requires a file, zim, or url')
def file_rows(self, config: dict) -> Tuple[Dict[tuple, dict], Set[tuple]]:
rows, invalid = dict(), set()
for data in config.get('playlists') or []:
if not (name := data.get('name')):
continue
# Positions follow the file's order; an item skipped now keeps its place for a later import.
for position, item in enumerate(data.get('items') or [], start=1):
try:
key = self.item_key(name, item)
except Exception as e:
logger.error(f'Skipping invalid item of playlist {name!r}: {item}', exc_info=e)
key = None
if key is None or key in rows:
if key:
logger.warning(f'Skipping duplicate item of playlist {name!r}: {item}')
# Counted as skipped (the config is not saved over it); a kind of None is never an item's key.
invalid.add((name, None, position, None))
continue
rows[key] = dict(position=position, title=item.get('title'))
return rows, invalid
def present(self, config: dict, file_rows: dict, invalid_keys: set) -> bool:
# A listed playlist without items has none: only an empty or missing playlists list removes nothing.
return any(i.get('name') for i in config.get('playlists') or [])
@staticmethod
def _query(session: Session, *columns):
from modules.zim.models import Zim
from wrolpi.files.models import FileGroup
from .models import Collection, CollectionItem
return session.query(*columns, Collection.name, CollectionItem.item_kind, FileGroup.primary_path, Zim.path,
CollectionItem.zim_entry, CollectionItem.url) \
.join(Collection, Collection.id == CollectionItem.collection_id) \
.outerjoin(FileGroup, FileGroup.id == CollectionItem.file_group_id) \
.outerjoin(Zim, Zim.id == CollectionItem.zim_id) \
.filter(Collection.kind == 'playlist')
@staticmethod
def _db_key(name, item_kind, primary_path, zim_path, zim_entry, url) -> Optional[tuple]:
if item_kind == 'file' and primary_path:
return name, 'file', media_relative(primary_path), None
elif item_kind == 'zim' and zim_path:
return name, 'zim', media_relative(zim_path), zim_entry
elif item_kind == 'url':
return name, 'url', url, None
# Not saved (see `CollectionItem.to_config`), so never "removed".
return None
def db_rows(self, session: Session) -> Dict[tuple, dict]:
from .models import CollectionItem
rows = dict()
for position, title, *values in self._query(session, CollectionItem.position, CollectionItem.title):
if key := self._db_key(*values):
rows[key] = dict(position=position, title=title)
return rows
def upsert(self, session: Session, rows: Dict[tuple, dict]):
from modules.zim.models import Zim
from wrolpi.files.models import FileGroup
from .models import Collection, CollectionItem
media_directory = get_media_directory()
collection_ids = dict(session.query(Collection.name, Collection.id).filter(
Collection.kind == 'playlist', Collection.name.in_(list({i[0] for i in rows}))))
file_group_ids = dict()
for paths in chunks({str(media_directory / i[2]) for i in rows if i[1] == 'file'}):
file_group_ids.update((media_relative(path), id_) for path, id_ in
session.query(FileGroup.primary_path, FileGroup.id)
.filter(FileGroup.primary_path.in_(paths)))
zim_ids = {media_relative(path): id_ for id_, path in session.query(Zim.id, Zim.path)} \
if any(i[1] == 'zim' for i in rows) else dict()
existing = {(i.collection_id, i.item_kind, i.file_group_id, i.zim_id, i.zim_entry, i.url): i for i in
session.query(CollectionItem).filter(CollectionItem.collection_id.in_(collection_ids.values()))}
new = list()
for key, row in rows.items():
name, kind, reference, zim_entry = key
values = dict(collection_id=collection_ids.get(name), item_kind=kind, file_group_id=None, zim_id=None,
zim_entry=None, url=None)
if kind == 'file':
values['file_group_id'] = file_group_ids.get(reference)
elif kind == 'zim':
values.update(zim_id=zim_ids.get(reference), zim_entry=zim_entry)
else:
values['url'] = reference
if not values['collection_id'] or (kind == 'file' and not values['file_group_id']) \
or (kind == 'zim' and not values['zim_id']):
logger.warning(f'Playlist {name!r}: cannot find {reference!r}, skipping')
continue
if item := existing.get(tuple(values.values())):
item.position, item.title = row['position'], row['title']
else:
new.append(dict(values, position=row['position'], title=row['title']))
if new:
# One statement for all new items (a rebuilt database has thousands). Not flushed: count it.
session.execute(insert(CollectionItem), new)
count_config_change(session, PlaylistsConfig.file_name)
session.flush()
def delete(self, session: Session, keys: Set[tuple]):
from .models import CollectionItem
for item, *values in self._query(session, CollectionItem) \
.filter(CollectionItem.collection_id.in_(self._collection_ids(session, keys))):
if self._db_key(*values) in keys:
session.delete(item)
@staticmethod
def _collection_ids(session: Session, keys) -> List[int]:
from .models import Collection
return [i for i, in session.query(Collection.id).filter(
Collection.kind == 'playlist', Collection.name.in_(list({i[0] for i in keys})))]
@dataclass
@ -48,6 +237,7 @@ class PlaylistsConfig(ConfigFile):
title: "The spot"
"""
file_name = 'playlists.yaml'
sections = (PlaylistsSection(), PlaylistItemsSection())
validator = PlaylistsConfigValidator
default_config = dict(version=0, playlists=[])
width = 120
@ -56,140 +246,83 @@ class PlaylistsConfig(ConfigFile):
def playlists(self) -> List[dict]:
return self._config.get('playlists', [])
def import_config(self, file: pathlib.Path = None, send_events=False):
"""Import playlists (and their ordered items) from the config into the database."""
def prepare(self, diffs: dict):
self._drop_unknown_tags(diffs['playlists'])
self._drop_missing_items(diffs['items'])
if not self.import_deletes:
self._merge_items(diffs['items'])
def _drop_unknown_tags(self, diff: SectionDiff):
"""A Tag that no longer exists (it was removed) is dropped, and the config is saved without it. Only once
tags.yaml was imported; until then its Tags may be missing."""
from wrolpi.tags import Tag, get_tags_config
rows = [*diff.added.values(), *diff.changed.values()]
if not (names := _playlist_tag_names(rows)) or not get_tags_config().successful_import:
return
with get_db_session() as session:
unknown = names - {i for i, in session.query(Tag.name).filter(Tag.name.in_(list(names)))}
for row in rows:
if row['tag_name'] in unknown:
logger.warning(f'Removing unknown Tag {row["tag_name"]!r} from a playlist')
row['tag_name'] = None
self._config_outdated = True
def _drop_missing_items(self, diff: SectionDiff):
"""An added item whose file or Zim is not in the database (not indexed yet, after a rebuild) is skipped
before the write; a later import (after the refresh) adds it."""
from modules.zim.models import Zim
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag
media_directory = get_media_directory()
files = {i[2] for i in diff.added if i[1] == 'file'}
zims = {i[2] for i in diff.added if i[1] == 'zim'}
known = set()
with get_db_session() as session:
for paths in chunks({str(media_directory / i) for i in files}):
known.update(('file', media_relative(i)) for i, in
session.query(FileGroup.primary_path).filter(FileGroup.primary_path.in_(paths)))
if zims:
known.update(('zim', media_relative(i)) for i, in session.query(Zim.path))
for key in list(diff.added):
if key[1] in ('file', 'zim') and (key[1], key[2]) not in known:
logger.warning(f'Playlist {key[0]!r}: {key[1]} not indexed, skipping {key[2]!r}')
del diff.added[key]
self.import_skipped += 1
from .models import Collection, CollectionItem
file = file or self.get_file()
file_str = str(self.get_relative_file())
# A missing config is NOT an empty config (mirrors ChannelsConfig). Deleting here would
# cascade: the sync prunes the on-disk playlist dirs and the next dump writes an empty
# config -- a single lost file would permanently destroy every playlist.
if not file.is_file():
logger.info(f'No playlists config file, skipping import: {file_str}')
self.successful_import = True
def _merge_items(self, diff: SectionDiff):
"""Without deletes, a playlist's items in the database keep their order, and the file's other items are
added after the item that precedes them in the file. Only titles are taken from the file's other items."""
section = PlaylistItemsSection()
if not (names := {i[0] for i in (*diff.added, *diff.changed)}):
return
with get_db_session() as session:
db_rows = section.db_rows(session)
file_rows, _ = section.file_rows(self._config)
for name in names:
merged = sorted((i for i in db_rows if i[0] == name), key=lambda i: db_rows[i]['position'])
previous = None
for key in sorted((i for i in file_rows if i[0] == name), key=lambda i: file_rows[i]['position']):
if key in db_rows:
previous = key
elif key in diff.added:
merged.insert(merged.index(previous) + 1 if previous else 0, key)
previous = key
for position, key in enumerate(merged, start=1):
if key in diff.added:
diff.added[key]['position'] = position
continue
row = dict(position=position, title=(diff.changed.get(key) or db_rows[key])['title'])
if row == db_rows[key]:
diff.changed.pop(key, None)
else:
diff.changed[key] = row
def import_config(self, file: pathlib.Path = None, send_events=False):
self._config_outdated = False
super().import_config(file, send_events)
decision = self.import_decision()
if decision == SAVE:
self.save_database_instead()
return
playlists_data = self._config.get('playlists', [])
# An empty playlists list never deletes DB records (mirrors ChannelsConfig). Deleting all
# playlists is done through the API, which dumps the resulting (empty) config itself.
if not playlists_data:
logger.info(f'No playlists in config, preserving existing DB playlists: {file_str}')
self.successful_import = True
return
try:
with get_db_session(commit=True) as session:
imported_names = set()
for data in playlists_data:
name = data.get('name')
if not name:
logger.error(f'Playlist config entry has no name, skipping: {data}')
self.import_skipped += 1
continue
collection = session.query(Collection).filter_by(
name=name, kind='playlist').one_or_none()
new = not collection
if new:
collection = Collection(name=name, kind='playlist')
session.add(collection)
collection.description = data.get('description')
# Optional custom directory (media-relative); None when not customized.
directory = data.get('directory')
collection.directory = (get_media_directory() / directory) if directory else None
tag_name = data.get('tag_name')
collection.tag = Tag.get_by_name(session, tag_name) if tag_name else None
session.flush([collection])
file_items = []
for item_data in (data.get('items') or []):
item = None
if 'file' in item_data:
path = get_media_directory() / item_data['file']
fg = session.query(FileGroup).filter_by(primary_path=str(path)).one_or_none()
if fg:
item = CollectionItem(collection_id=collection.id, item_kind='file',
file_group_id=fg.id)
else:
logger.warning(f'Playlist {name!r}: file not indexed, skipping '
f'{item_data["file"]!r}')
self.import_skipped += 1
elif 'zim' in item_data:
zpath = get_media_directory() / item_data['zim']
zim = session.query(Zim).filter_by(path=str(zpath)).one_or_none()
if zim:
item = CollectionItem(collection_id=collection.id, item_kind='zim',
zim_id=zim.id, zim_entry=item_data.get('entry'))
else:
logger.warning(f'Playlist {name!r}: zim not found, skipping '
f'{item_data["zim"]!r}')
self.import_skipped += 1
elif 'url' in item_data:
item = CollectionItem(collection_id=collection.id, item_kind='url',
url=item_data['url'])
if item is not None:
item.title = item_data.get('title')
file_items.append(item)
if decision == IMPORT or new:
# Rebuild items to match the config's order exactly.
for item in list(collection.items):
session.delete(item)
session.flush()
items = file_items
else:
# Without deletes, keep the database's items in their order; add the file's other items
# after the item that precedes them in the file.
items = sorted(collection.items, key=lambda i: i.position)
existing = {_item_key(i): i for i in items}
previous = None
for item in file_items:
if found := existing.get(_item_key(item)):
previous = found
continue
items.insert(items.index(previous) + 1 if previous else 0, item)
previous = item
for position, item in enumerate(items, start=1):
item.position = position
session.add(item)
session.flush()
imported_names.add(name)
# Delete playlists that are no longer present in the config.
for collection in session.query(Collection).filter_by(kind='playlist').all() \
if decision == IMPORT else []:
if collection.name not in imported_names:
logger.info(f'Deleting playlist {collection.name!r} (no longer in config)')
session.delete(collection)
logger.info(f'Imported {len(imported_names)} playlists from {file_str}')
self.record_import_version()
self.successful_import = True
except Exception as e:
self.successful_import = False
message = f'Failed to import {file_str} config!'
logger.error(message, exc_info=e)
if send_events:
Events.send_config_import_failed(message)
raise
def after_import(self, diffs: dict):
if self._config_outdated:
self.background_dump.activate_switch()
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
"""Preview which playlists a backup import would add/remove (keyed by playlist name)."""
@ -302,7 +435,7 @@ def file_groups_moved(session, file_group_ids):
playlists.yaml references file items by path, so it must be re-saved (and the playlist
directories re-synced) or the next import will not find the file and will drop the item.
tags.yaml saves tagged files by path too; the move (often raw SQL) is counted as its change.
tags.yaml saves tagged files by path too. The move (often raw SQL) is counted as each config's change.
`file_group_ids` may be an iterable of ids or a select of ids."""
from .models import Collection, CollectionItem
@ -316,6 +449,7 @@ def file_groups_moved(session, file_group_ids):
.filter(Collection.kind == 'playlist', CollectionItem.file_group_id.in_(file_group_ids)) \
.first()
if in_playlist:
count_config_change(session, PlaylistsConfig.file_name)
save_playlists_config.activate_switch()
from .sync import sync_playlists_directory
sync_playlists_directory.activate_switch()

22
wrolpi/collections/test/test_playlists_config.py
View file

@ -392,3 +392,25 @@ async def test_import_without_deletes_keeps_database_items(test_session, test_di
test_session.expire_all()
collection = test_session.query(Collection).filter_by(name='Mix', kind='playlist').one()
assert [i.url for i in collection.items] == ['/b', '/a', '/c']
@pytest.mark.asyncio
async def test_import_without_deletes_keeps_database_order(test_session, test_directory, playlists_config):
"""Without a database version, a file that only reorders a playlist's items (and lacks one the database has)
leaves the database's order as it is."""
collection = Collection(name='Mix', kind='playlist')
test_session.add(collection)
test_session.flush([collection])
collection.add_url(test_session, '/a', title='A')
collection.add_url(test_session, '/x', title='X') # Added after the file was saved.
collection.add_url(test_session, '/b', title='B')
test_session.commit()
playlists_config.get_file().write_text(yaml.dump(dict(version=1, playlists=[
dict(name='Mix', items=[dict(url='/b', title='B'), dict(url='/a', title='A')]),
])))
playlists_config.import_config()
test_session.expire_all()
collection = test_session.query(Collection).filter_by(name='Mix', kind='playlist').one()
assert [(i.url, i.position) for i in collection.items] == [('/a', 1), ('/x', 2), ('/b', 3)]

250
wrolpi/collections/test/test_playlists_config_import.py Normal file
View file

@ -0,0 +1,250 @@
"""playlists.yaml imports by diffing its playlists and their items against the database, and writing only the
differences (see wrolpi.config_import). An item is keyed by what it points to, so moving it changes only its
position."""
import pytest
import yaml
from wrolpi.collections.config import get_playlists_config
from wrolpi.collections.models import Collection, CollectionItem
from wrolpi.config_version import get_config_version
from wrolpi.conftest import config_was_imported
from wrolpi.files.models import FileGroup
from wrolpi.test.test_config_import import edit_config, statements, writes
ONE = ['url:/one', 'file:a.txt', 'zim:home']
@pytest.fixture
async def saved_playlists(test_session, test_directory, async_client, make_files_structure, test_zim, tag_factory,
await_switches):
"""Playlist `One` (a URL, a file, and a Zim entry, tagged `one`) and `Two` (a URL), saved to the config (which
records its version)."""
from copy import deepcopy
config = get_playlists_config()
config._config = deepcopy(config.default_config)
tag = await tag_factory('one')
a, = make_files_structure({'a.txt': 'a'})
a = FileGroup.from_paths(test_session, a)
one = Collection(name='One', kind='playlist', description='The first', tag_id=tag.id)
two = Collection(name='Two', kind='playlist')
test_session.add_all([a, one, two])
test_session.flush([a, one, two])
one.add_url(test_session, '/one', title='One URL')
one.add_file_group(a, session=test_session)
one.add_zim_entry(test_session, test_zim.id, 'home')
two.add_url(test_session, '/two')
test_session.commit()
config.dump_config()
await await_switches()
assert get_config_version(test_session, 'playlists.yaml') is not None
return config.get_file()
def items(test_session, name='One') -> list:
"""`kind:reference` of each item of the playlist, in order."""
test_session.expire_all()
collection = test_session.query(Collection).filter_by(name=name, kind='playlist').one()
return [f'{i.item_kind}:{i.url or i.zim_entry or i.file_group.primary_path.name}' for i in collection.items]
def item_ids(test_session, name='One') -> dict:
test_session.expire_all()
collection = test_session.query(Collection).filter_by(name=name, kind='playlist').one()
return {i.id for i in collection.items}
def edit_playlist(path, name, change):
def change_data(data):
change(next(i for i in data['playlists'] if i['name'] == name))
edit_config(path, change_data)
@pytest.mark.asyncio
async def test_unchanged_import_writes_nothing(test_session, saved_playlists):
"""Importing a config the database already matches opens no write transaction."""
config = get_playlists_config()
with statements() as collected:
config.import_config()
assert config.successful_import is True
assert writes(collected) == []
# The version, then one snapshot per section.
assert len(collected) <= 4, collected
assert items(test_session) == ONE
@pytest.mark.asyncio
async def test_reordered_items_keep_their_rows(test_session, saved_playlists):
"""Moving and inserting items changes only positions: every item keeps its row (and ID)."""
before = item_ids(test_session)
def change(playlist):
playlist['items'].reverse()
playlist['items'].insert(0, dict(url='/new', title='New'))
edit_playlist(saved_playlists, 'One', change)
config = get_playlists_config()
with statements() as collected:
config.import_config()
assert config.successful_import is True
assert items(test_session) == ['url:/new', 'zim:home', 'file:a.txt', 'url:/one']
assert before < item_ids(test_session)
assert not [i for i in writes(collected) if i.upper().startswith('DELETE FROM COLLECTION_ITEM')]
@pytest.mark.asyncio
async def test_removed_rows_are_deleted(test_session, saved_playlists):
"""A removed item and a removed playlist are deleted; a changed description and title are saved."""
def change(data):
data['playlists'] = [i for i in data['playlists'] if i['name'] != 'Two']
one = data['playlists'][0]
one['description'] = 'Changed'
one['items'] = [i for i in one['items'] if 'zim' not in i]
one['items'][0]['title'] = 'Renamed'
edit_config(saved_playlists, change)
config = get_playlists_config()
config.import_config()
assert config.successful_import is True
assert {i.name for i in test_session.query(Collection).filter_by(kind='playlist')} == {'One'}
assert items(test_session) == ['url:/one', 'file:a.txt']
one = test_session.query(Collection).filter_by(name='One', kind='playlist').one()
assert (one.description, one.items[0].title) == ('Changed', 'Renamed')
@pytest.mark.asyncio
async def test_unindexed_file_is_skipped_without_a_write(test_session, saved_playlists):
"""An item whose file is not indexed (yet) is skipped before the write, and keeps its place: the items after it
keep their file positions. The next import, with nothing else to do, writes nothing."""
edit_playlist(saved_playlists, 'Two', lambda playlist: playlist['items'].insert(0, dict(file='later.txt')))
config = get_playlists_config()
config.import_config()
test_session.expire_all()
assert [i.position for i in test_session.query(Collection).filter_by(name='Two').one().items] == [2]
with statements() as collected:
config.import_config()
assert config.successful_import is True
assert config.import_skipped == 1
assert writes(collected) == []
assert items(test_session, 'Two') == ['url:/two']
@pytest.mark.asyncio
async def test_new_playlist_is_created_with_its_items(test_session, saved_playlists):
edit_config(saved_playlists, lambda data: data['playlists'].append(dict(
name='Three', description='New', tag_name='one', items=[dict(url='/three'), dict(file='a.txt')])))
get_playlists_config().import_config()
three = test_session.query(Collection).filter_by(name='Three', kind='playlist').one()
assert (three.description, three.tag_name) == ('New', 'one')
assert items(test_session, 'Three') == ['url:/three', 'file:a.txt']
@pytest.mark.asyncio
async def test_unknown_tag_is_dropped(test_session, saved_playlists, test_tags_config, await_switches):
"""Once tags.yaml is imported, a playlist's Tag that no longer exists is dropped and the config is saved without
it: the next import writes nothing."""
from wrolpi.tags import get_tags_config
get_tags_config().dump_config()
get_tags_config().import_config()
edit_playlist(saved_playlists, 'One', lambda playlist: playlist.update(tag_name='gone'))
config = get_playlists_config()
config.import_config()
await await_switches()
assert test_session.query(Collection).filter_by(name='One', kind='playlist').one().tag_id is None
assert 'gone' not in saved_playlists.read_text()
with statements() as collected:
config.import_config()
assert writes(collected) == []
@pytest.mark.asyncio
async def test_moved_playlist_file_counts(test_session, test_directory, saved_playlists, make_files_structure):
"""playlists.yaml saves file items by path: moving a FileGroup in a playlist makes the database newer than the
file, so a restart before the save does not drop the item. Moving another FileGroup does not."""
before = get_config_version(test_session, 'playlists.yaml')
a = test_session.query(FileGroup).filter_by(primary_path=test_directory / 'a.txt').one()
a.move(test_directory / 'moved.txt')
test_session.commit()
assert get_config_version(test_session, 'playlists.yaml') == before + 1
other = FileGroup.from_paths(test_session, *make_files_structure({'other.txt': 'other'}))
test_session.commit()
other.move(test_directory / 'other-moved.txt')
test_session.commit()
assert get_config_version(test_session, 'playlists.yaml') == before + 1
@pytest.mark.asyncio
async def test_bulk_moved_playlist_file_counts(test_session, test_directory, saved_playlists):
"""A move written with raw SQL counts too."""
from wrolpi.files.lib import _bulk_update_file_groups_db
before = get_config_version(test_session, 'playlists.yaml')
_bulk_update_file_groups_db(test_session, {test_directory / 'a.txt': test_directory / 'moved/a.txt'})
test_session.commit()
assert get_config_version(test_session, 'playlists.yaml') > before
@pytest.mark.asyncio
async def test_refresh_changing_primary_file_counts(test_session, test_directory, make_files_structure, video_bytes,
refresh_files):
"""A refresh that changes a playlist FileGroup's primary file counts as a change of the item's path."""
make_files_structure({'video.mp4': video_bytes, 'video.jpg': 'poster'})
await refresh_files()
playlist = Collection(name='Videos', kind='playlist')
test_session.add(playlist)
test_session.flush([playlist])
playlist.add_file_group(test_session.query(FileGroup).one(), session=test_session)
test_session.commit()
config_was_imported(test_session, 'playlists.yaml')
test_session.commit()
(test_directory / 'video.mp4').unlink()
await refresh_files()
test_session.expire_all()
assert test_session.query(FileGroup).one().primary_path != test_directory / 'video.mp4'
assert get_config_version(test_session, 'playlists.yaml') > 0
@pytest.mark.asyncio
async def test_empty_items_are_removed(test_session, saved_playlists):
"""A listed playlist with no items has its items removed, even when no playlist in the file has an item (only
an empty or missing playlists list deletes nothing)."""
def change(data):
for playlist in data['playlists']:
playlist['items'] = []
edit_config(saved_playlists, change)
config = get_playlists_config()
config.import_config()
assert config.successful_import is True
assert items(test_session, 'One') == []
assert items(test_session, 'Two') == []
@pytest.mark.asyncio
async def test_invalid_items_are_skipped(test_session, saved_playlists):
"""An item that cannot be read (a Zim item without an entry), or that repeats another, is counted as skipped,
so the config is not saved over it."""
def change(playlist):
playlist['items'].append(dict(zim='other.zim'))
playlist['items'].append(dict(url='/one'))
edit_playlist(saved_playlists, 'One', change)
config = get_playlists_config()
config.import_config()
assert config.import_skipped == 2
assert items(test_session) == ONE

7
wrolpi/common.py
View file

@ -844,10 +844,12 @@ class ConfigFile:
file_rows = {i.name: i.file_rows(self._config) for i in self.sections}
with get_db_session() as session:
db_rows = {i.name: i.db_rows(session) for i in self.sections}
diffs = {i.name: diff_section(*file_rows[i.name], db_rows[i.name], deletes=decision == IMPORT, same=i.same)
diffs = {i.name: diff_section(*file_rows[i.name], db_rows[i.name], deletes=decision == IMPORT, same=i.same,
present=i.present(self._config, *file_rows[i.name]))
for i in self.sections}
# `prepare` may skip rows it cannot apply.
self.import_skipped = sum(len(invalid) for _, invalid in file_rows.values())
self.import_deletes = decision == IMPORT
if any(diffs.values()):
self.prepare(diffs)
@ -889,7 +891,8 @@ class ConfigFile:
def prepare(self, diffs: dict):
"""Slow work the write needs (files, network), run with no transaction open, and only when there are
differences. `diffs` maps each section's name to its `SectionDiff`; it may drop rows it cannot apply
(count them in `import_skipped`), or add what it found to a row for `upsert`.
(count them in `import_skipped`), or add what it found to a row for `upsert`. `import_deletes`: whether
this import deletes what the file no longer lists.
It must be safe to repeat and to abandon: when the database changes before the write, the import decides
again (up to three passes), and the diffs it was given may never be written (the write fails, or the next

24
wrolpi/config_import.py
View file

@ -6,11 +6,14 @@ Rows are compared as plain data keyed the way the file names them (a URL, a name
ORM objects and the read can finish before any slow preparation starts.
"""
import operator
import pathlib
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, Hashable, Iterable, List, Set, Tuple
from sqlalchemy.orm import Session
from wrolpi.common import get_media_directory
@dataclass
class SectionDiff:
@ -46,6 +49,11 @@ class ConfigSection:
deleted since the snapshot."""
raise NotImplementedError()
def present(self, config: dict, file_rows: dict, invalid_keys: set) -> bool:
"""Whether the file lists this section at all; an empty section cannot be told apart from a lost or blanked
file, so it removes nothing."""
return bool(file_rows or invalid_keys)
def same(self, file_row: dict, db_row: dict) -> bool:
"""Whether a file row is already in the database."""
return file_row == db_row
@ -57,20 +65,30 @@ class ConfigSection:
def diff_section(file_rows: Dict[Hashable, dict], invalid_keys: Set[Hashable], db_rows: Dict[Hashable, dict],
deletes: bool, same: Callable[[dict, dict], bool] = operator.eq) -> SectionDiff:
deletes: bool, same: Callable[[dict, dict], bool] = operator.eq, present: bool = None) \
-> SectionDiff:
"""Compare a section's file rows with its database rows. `deletes`: rows the file no longer lists are removed.
An empty section never removes anything: it cannot be told apart from a lost or blanked file."""
A section that is not `present` (by default, an empty one) never removes anything: it cannot be told apart from
a lost or blanked file."""
diff = SectionDiff()
for key, row in file_rows.items():
if key not in db_rows:
diff.added[key] = row
elif not same(row, db_rows[key]):
diff.changed[key] = row
if deletes and (file_rows or invalid_keys):
if present is None:
present = bool(file_rows or invalid_keys)
if deletes and present:
diff.removed = db_rows.keys() - file_rows.keys() - invalid_keys
return diff
def media_relative(path) -> str:
"""A path as a config names it: relative to the media directory."""
path = pathlib.Path(path)
return str(path.relative_to(get_media_directory()) if path.is_absolute() else path)
def chunks(items: Iterable[Any], size: int = 500) -> Iterable[List[Any]]:
"""Split `items` for `IN (...)` queries (SQLite limits the parameters of one statement)."""
items = list(items)

14
wrolpi/files/models.py
View file

@ -905,6 +905,20 @@ def _tagged_file_group_moved(file_group: FileGroup) -> bool:
register_ownership('tags.yaml', FileGroup, columns={'primary_path'}, row=_tagged_file_group_moved)
def _playlist_file_group_moved(file_group: FileGroup) -> bool:
"""An existing FileGroup in a playlist changed path (playlists.yaml saves file items by path)."""
if not inspect(file_group).attrs.primary_path.history.deleted:
return False
session = Session.object_session(file_group)
return session.connection().execute(
text('SELECT 1 FROM collection_item JOIN collection ON collection.id = collection_item.collection_id'
" WHERE collection_item.file_group_id = :id AND collection.kind = 'playlist' LIMIT 1"),
{'id': file_group.id}).first() is not None
register_ownership('playlists.yaml', FileGroup, columns={'primary_path'}, row=_playlist_file_group_moved)
class Directory(ModelHelper, Base):
"""A representation of a file directory in the media directory."""
__tablename__ = 'directory'

13
wrolpi/files/worker.py
View file

@ -759,6 +759,19 @@ class FileWorker(FileMoveMixin, FileReorganizeMixin):
{'updates': updates})
if curs.fetchone():
count_config_change_curs(curs, 'tags.yaml')
# playlists.yaml saves file items by path too.
curs.execute(
'''SELECT 1
FROM collection_item
JOIN collection ON collection.id = collection_item.collection_id
JOIN file_group ON file_group.id = collection_item.file_group_id
JOIN json_each(:updates) AS v ON file_group.id = json_extract(v.value, '$[0]')
WHERE collection.kind = 'playlist'
AND file_group.primary_path != json_extract(v.value, '$[1]')
LIMIT 1''',
{'updates': updates})
if curs.fetchone():
count_config_change_curs(curs, 'playlists.yaml')
curs.execute(
'''UPDATE file_group
SET primary_path = v.new_path

12
wrolpi/tags.py
View file

@ -14,7 +14,7 @@ from sqlalchemy.orm import Mapped, relationship, Session, load_only
from wrolpi import dates, flags
from wrolpi.common import loaded, ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, \
get_relative_to_media_directory, is_valid_hex_color, walk, INVALID_FILE_CHARS, get_wrolpi_config
from wrolpi.config_import import ConfigSection, SectionDiff, chunks
from wrolpi.config_import import ConfigSection, SectionDiff, chunks, media_relative
from wrolpi.config_version import count_config_change, register_ownership
from wrolpi.dates import TZDateTime
from wrolpi.db import get_db_session, named_placeholders, session_curs, serializer
@ -307,12 +307,6 @@ def _utc_or_none(value) -> Optional[datetime]:
return value.astimezone(pytz.UTC) if value.tzinfo else value.replace(tzinfo=pytz.UTC)
def _relative_path(path) -> str:
"""A path as the config names it: relative to the media directory."""
path = pathlib.Path(path)
return str(path.relative_to(get_media_directory()) if path.is_absolute() else path)
def without_tag_names(download_tag_names: Optional[List[str]], settings: Optional[dict], names: Set[str]) \
-> Tuple[Optional[List[str]], Optional[dict]]:
"""A Download's tag_names and settings without these Tag names, in the settings' tag_names and channel_tag_name
@ -398,7 +392,7 @@ class TagFilesSection(ConfigSection):
for item in config.get('tag_files') or []:
try:
tag_name, path, *rest = item
key = (tag_name, _relative_path(path))
key = (tag_name, media_relative(path))
except Exception as e:
logger.error(f'Skipping invalid tagged file in config: {item}', exc_info=e)
continue
@ -497,7 +491,7 @@ class TagZimsSection(ConfigSection):
for item in config.get('tag_zims') or []:
try:
tag_name, zim_path, zim_entry, *rest = item
key = (tag_name, _relative_path(zim_path), zim_entry)
key = (tag_name, media_relative(zim_path), zim_entry)
except Exception as e:
logger.error(f'Skipping invalid tagged Zim entry in config: {item}', exc_info=e)
continue