From d11b72bbd127237f98132805519939c339ea15a8 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:06:22 -0700 Subject: [PATCH] A file the sync replaces keeps its Tags Directory and playlist links (#986) * A file the sync replaces keeps its Tags Directory and playlist links A secondary replaces a changed file with a new inode, so the file's hard links in the Tags Directory and the playlists directory were left holding the old version. Both directories repair a link that is still wanted, but refuse to delete one with no other link. When the primary also changed the file's tags, or reordered the playlist, in the same sync, the old link was no longer wanted and could never be deleted. After each page's downloads, every link to a replaced version in those two directories now becomes a link to the new version. Co-Authored-By: Claude Fable 5.2 * A replaced file's links move before the new version takes its place Review of the first version found three gaps: - A playlist with its own directory keeps its links outside the playlists directory; those were not looked at. - The links were repaired after a whole page of downloads, so a sync that stopped mid-page left the files it had already replaced with their old links, and a later sync saw nothing to do. - Two library files that are hard links of each other share an inode, so the link of the one that did not change was pointed at the one that did. The links of a page's files are now found before its downloads, in the Tags Directory, the playlists directory and each playlist's own directory, and only by the name those give a link. Each file's links are moved to the new version just before it is renamed into place. The temporary name is short, so a long file name still fits. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: Claude Fable 5.2 --- wrolpi/sync/lib.py | 106 +++++++++++++++++++- wrolpi/sync/test/test_engine.py | 169 +++++++++++++++++++++++++++++++- 2 files changed, 270 insertions(+), 5 deletions(-) diff --git a/wrolpi/sync/lib.py b/wrolpi/sync/lib.py index 240210eb..814380f4 100644 --- a/wrolpi/sync/lib.py +++ b/wrolpi/sync/lib.py @@ -26,10 +26,11 @@ import re import shutil import ssl import time +import uuid from dataclasses import dataclass from datetime import datetime from http import HTTPStatus -from typing import AsyncGenerator, List, Optional +from typing import AsyncGenerator, Callable, Dict, List, Optional from urllib.parse import quote import aiohttp @@ -38,13 +39,17 @@ from sqlalchemy import func, select from sqlalchemy.orm import Session from wrolpi import flags +from wrolpi.collections.models import Collection +from wrolpi.collections.sync import get_playlists_directory, MANAGED_NAME from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary, wrol_mode_enabled, \ get_config_by_file_name, SYNCED_CONFIG_FILE_NAMES from wrolpi.dates import now, is_within_daily_window +from wrolpi.db import get_db_session from wrolpi.errors import InvalidConfig from wrolpi.events import Events from wrolpi.sync.config import get_sync_config from wrolpi.sync.status import get_sync_status, update_sync_status +from wrolpi.tags import get_tags_directory from wrolpi.vars import DEFAULT_HTTP_HEADERS, PYTEST from wrolpi.version import __version__ @@ -402,10 +407,11 @@ class PrimaryClient: async def download_file(client: PrimaryClient, file: ManifestFile, destination: pathlib.Path, - on_bytes=None) -> int: + on_bytes=None, before_replace: Callable[[pathlib.Path], None] = None) -> int: """Download one file to `destination` with the primary's modification time. Returns the bytes received. - A partial file left by an earlier attempt is resumed.""" + A partial file left by an earlier attempt is resumed. `before_replace` is called with the complete new version + just before it takes the place of `destination`.""" partial = partial_path(destination) destination.parent.mkdir(parents=True, exist_ok=True) @@ -449,10 +455,95 @@ async def download_file(client: PrimaryClient, file: ManifestFile, destination: if (size := partial.stat().st_size) != file.size: raise SyncError(f'{file.path} is {size} bytes, the primary said {file.size}') os.utime(partial, (file.mtime, file.mtime)) + if before_replace: + before_replace(partial) os.replace(partial, destination) return received +def link_directories() -> List[tuple]: + """Where hard links of library files are kept, each as (directory, recursive): the Tags Directory, the + playlists directory, and the directory of every playlist that has its own.""" + directories = [(get_tags_directory(), True), (get_playlists_directory(), True)] + with get_db_session() as session: + custom = session.execute( + select(Collection.directory).where(Collection.kind == 'playlist', Collection.directory.isnot(None)) + ).scalars().all() + # A playlist's own directory may be any folder of the library; only the links directly in it are its own. + directories.extend((pathlib.Path(i), False) for i in custom) + return directories + + +def _is_link_name(link_name: str, file_name: str) -> bool: + """True if `link_name` is what the Tags Directory (the file's name) or a playlist (the name after its position) + calls a link to a file named `file_name`.""" + return link_name == file_name or bool(MANAGED_NAME.match(link_name)) and link_name[5:] == file_name + + +def find_links(files: List[pathlib.Path]) -> Dict[pathlib.Path, List[pathlib.Path]]: + """The Tags Directory and playlist links of each of `files`: the files in `link_directories` that are the same + inode as the file and are named for it. A file without other hard links has none and is left out.""" + inodes = dict() + for file in files: + try: + stat = file.lstat() + except FileNotFoundError: + continue + if stat.st_nlink > 1: + # Library files may be hard links of each other, so several files can share an inode. + inodes.setdefault((stat.st_dev, stat.st_ino), []).append(file) + links = dict() + if not inodes: + return links + + seen = set() + for directory, recursive in link_directories(): + roots = (i[0] for i in os.walk(directory)) if recursive else (str(directory),) + for root in roots: + if root in seen: + continue + seen.add(root) + try: + names = os.listdir(root) + except OSError: + continue + for name in names: + path = pathlib.Path(root, name) + try: + stat = path.lstat() + except OSError: + continue + for file in inodes.get((stat.st_dev, stat.st_ino), ()): + if _is_link_name(name, file.name): + links.setdefault(file, []).append(path) + return links + + +def move_links(links: List[pathlib.Path], old_version: pathlib.Path, new_version: pathlib.Path): + """Make each of `links` that is still a link to `old_version` a link to `new_version`. + + The Tags Directory and a playlist refuse to delete a file that has no other link, because it may be a last + copy. A link left on a replaced version is exactly that: once it is no longer wanted (the primary changed the + file's tags, or reordered the playlist) it would stay forever.""" + try: + old = old_version.lstat() + except FileNotFoundError: + return + for link in links: + # Short, so it fits wherever the link's own name does. + temporary = link.with_name(f'.{uuid.uuid4().hex}.sync') + try: + stat = link.lstat() + if (stat.st_dev, stat.st_ino) != (old.st_dev, old.st_ino): + continue + # Linked beside it, then renamed over it: the link is never missing. + os.link(new_version, temporary) + os.replace(temporary, link) + except OSError as e: + logger.error(f'Failed to move the link {repr(str(link))}: {e}') + temporary.unlink(missing_ok=True) + + # --------------------------------------------------------------------------------------------------------------------- # Secondary: configs. @@ -790,6 +881,9 @@ async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]: page_downloaded = list() if wanted and not disk_full: semaphore = asyncio.Semaphore(DOWNLOAD_CONCURRENCY) + # A file's links move to its new version before the new version takes its place, so a sync that is + # interrupted in between downloads nothing again and still leaves no link on the old version. + links = await asyncio.to_thread(find_links, [i for _, i in wanted]) async def download(file_: ManifestFile, destination_: pathlib.Path): nonlocal disk_full @@ -797,7 +891,11 @@ async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]: if disk_full: return None try: - await download_file(client, file_, destination_, count_bytes) + before_replace = None + if destination_ in links: + def before_replace(new_version: pathlib.Path): + move_links(links[destination_], destination_, new_version) + await download_file(client, file_, destination_, count_bytes, before_replace) downloaded.append(destination_) return destination_ except SyncDiskFull as e: diff --git a/wrolpi/sync/test/test_engine.py b/wrolpi/sync/test/test_engine.py index 218d2294..fc5f886c 100644 --- a/wrolpi/sync/test/test_engine.py +++ b/wrolpi/sync/test/test_engine.py @@ -5,6 +5,8 @@ from collections import namedtuple import pytest import yaml +from wrolpi.collections.models import Collection +from wrolpi.collections.sync import get_playlists_directory, _safe_unlink_managed from wrolpi.common import get_wrolpi_config, set_test_media_directory from wrolpi.db import get_db_session from wrolpi.errors import InvalidConfig @@ -13,7 +15,7 @@ from wrolpi.sync import lib from wrolpi.sync.config import get_sync_config from wrolpi.sync.conftest import manifest_page from wrolpi.sync.status import get_sync_status, update_sync_status -from wrolpi.tags import get_tags_config, Tag, TagFile +from wrolpi.tags import get_tags_config, Tag, TagFile, _delete_extra_tags_directory_paths @pytest.mark.parametrize('path, allowed', [ @@ -273,6 +275,171 @@ async def test_run_sync(async_client, test_session, primary, secondary_directory assert not [i for i in primary.state['requests'] if not i[0].startswith('/media/config/')] +@pytest.mark.asyncio +async def test_run_sync_replaced_file_keeps_its_links(async_client, primary, secondary_directory, make_secondary, + fake_primary_api): + """The Tags Directory and playlists links of a file the sync replaces are links to the new version. A link to + the old version would be the old version's last copy, which neither directory ever deletes.""" + changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) + same = write_primary_file(primary, 'docs/same.txt', b'unchanged') + fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime), (same.path, same.size, same.mtime))], + primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + # The secondary has an older version of one file, linked into a tag and a playlist. + notes = secondary_directory / 'docs/notes.txt' + unchanged = secondary_directory / 'docs/same.txt' + notes.parent.mkdir(parents=True) + notes.write_bytes(b'old notes') + unchanged.write_bytes(b'unchanged') + os.utime(unchanged, (same.mtime, same.mtime)) + tag_link = secondary_directory / 'tags/one/notes.txt' + playlist_link = get_playlists_directory() / 'Reading/0001_notes.txt' + unchanged_link = secondary_directory / 'tags/one/same.txt' + # The only copy of a file that someone put in the Tags Directory themselves. + stranger = secondary_directory / 'tags/one/stranger.txt' + for source, link in ((notes, tag_link), (notes, playlist_link), (unchanged, unchanged_link)): + link.parent.mkdir(parents=True, exist_ok=True) + link.hardlink_to(source) + stranger.write_bytes(b'old notes') + + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert status['files_downloaded'] == 1 + assert notes.read_bytes() == b'the new notes' + for link in (tag_link, playlist_link): + assert link.samefile(notes), f'{link} is still the old version' + assert notes.stat().st_nlink == 3 + assert unchanged_link.samefile(unchanged) + assert stranger.read_bytes() == b'old notes' and stranger.stat().st_nlink == 1 + assert not [i for i in secondary_directory.rglob('.*') if i.is_file()], 'no temporary file is left behind' + + # The links are no longer wanted (the tag was removed, the playlist reordered): both directories can delete them. + _delete_extra_tags_directory_paths(secondary_directory / 'tags', [unchanged_link, stranger]) + assert not tag_link.exists() + assert _safe_unlink_managed(playlist_link) + assert notes.read_bytes() == b'the new notes' and notes.stat().st_nlink == 1 + + +@pytest.mark.asyncio +async def test_run_sync_replaced_file_in_a_playlist_with_its_own_directory( + async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api): + """A playlist may keep its links in any folder of the library; a replaced file's link there is moved too, and + the folder's other files are left alone.""" + changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) + fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + notes = secondary_directory / 'docs/notes.txt' + notes.parent.mkdir(parents=True) + notes.write_bytes(b'old notes') + directory = secondary_directory / 'my lists/Reading' + (directory / 'deeper').mkdir(parents=True) + link = directory / '0001_notes.txt' + link.hardlink_to(notes) + # A hard link of the file that the playlist did not make: not its name, or not directly in its folder. + renamed = directory / 'my copy of the notes.txt' + deeper = directory / 'deeper/0001_notes.txt' + renamed.hardlink_to(notes) + deeper.hardlink_to(notes) + test_session.add(Collection(name='Reading', kind='playlist', directory=directory)) + test_session.commit() + + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert link.samefile(notes) and link.read_bytes() == b'the new notes' + assert renamed.read_bytes() == b'old notes' and deeper.read_bytes() == b'old notes' + assert _safe_unlink_managed(link) + + +@pytest.mark.asyncio +async def test_run_sync_replaced_file_with_a_hard_linked_twin(async_client, primary, secondary_directory, + make_secondary, fake_primary_api): + """Two library files may be hard links of each other. When one of them changes, only links named for it move; + its twin's link is still its twin.""" + changed = write_primary_file(primary, 'docs/a.txt', b'the new a', mtime=1_700_000_500.0) + fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + a, b = secondary_directory / 'docs/a.txt', secondary_directory / 'docs/b.txt' + a.parent.mkdir(parents=True) + a.write_bytes(b'shared') + b.hardlink_to(a) + a_link, b_link = secondary_directory / 'tags/one/a.txt', secondary_directory / 'tags/one/b.txt' + a_link.parent.mkdir(parents=True) + a_link.hardlink_to(a) + b_link.hardlink_to(b) + + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert a.read_bytes() == b'the new a' and a_link.samefile(a) + assert b.read_bytes() == b'shared' and b_link.samefile(b) + + +@pytest.mark.asyncio +async def test_run_sync_interrupted_before_a_file_is_replaced(async_client, primary, secondary_directory, + make_secondary, fake_primary_api, monkeypatch): + """A sync that stops between downloading a new version and putting it in place has left no link on the old + version once a later sync finishes the job, although that sync downloads nothing.""" + changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) + fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + notes = secondary_directory / 'docs/notes.txt' + notes.parent.mkdir(parents=True) + notes.write_bytes(b'old notes') + link = secondary_directory / 'tags/one/notes.txt' + link.parent.mkdir(parents=True) + link.hardlink_to(notes) + + replace = os.replace + + def power_cut(source, destination): + if pathlib.Path(destination) == notes: + raise OSError('the power went out') + return replace(source, destination) + + monkeypatch.setattr(lib.os, 'replace', power_cut) + status = await lib.run_sync() + assert status['files_failed'] == 1 + assert notes.read_bytes() == b'old notes', 'the file was not replaced' + monkeypatch.setattr(lib.os, 'replace', replace) + + primary.state['requests'].clear() + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert status['files_downloaded'] == 1 and status['bytes_downloaded'] == 0 + assert notes.read_bytes() == b'the new notes' + assert link.samefile(notes) and notes.stat().st_nlink == 2 + assert not [i for i in secondary_directory.rglob('.*') if i.is_file()] + + +def test_move_links_of_a_long_name(test_session, test_directory): + """A link whose name is as long as a file name may be is still moved.""" + name = 'n' * 240 + '.txt' + old, new = test_directory / 'docs' / name, test_directory / 'docs/new version' + old.parent.mkdir() + old.write_bytes(b'old') + new.write_bytes(b'new') + link = get_playlists_directory() / 'Reading' / f'0001_{name}' + link.parent.mkdir(parents=True) + link.hardlink_to(old) + + assert lib.find_links([old]) == {old: [link]} + lib.move_links([link], old, new) + + assert link.samefile(new) + assert [i.name for i in link.parent.iterdir()] == [link.name] + + @pytest.mark.asyncio async def test_run_sync_refuses_unsafe_paths(async_client, primary, secondary_directory, make_secondary, fake_primary_api):