diff --git a/wrolpi/sync/lib.py b/wrolpi/sync/lib.py index 240210eb..814380f4 100644 --- a/wrolpi/sync/lib.py +++ b/wrolpi/sync/lib.py @@ -26,10 +26,11 @@ import re import shutil import ssl import time +import uuid from dataclasses import dataclass from datetime import datetime from http import HTTPStatus -from typing import AsyncGenerator, List, Optional +from typing import AsyncGenerator, Callable, Dict, List, Optional from urllib.parse import quote import aiohttp @@ -38,13 +39,17 @@ from sqlalchemy import func, select from sqlalchemy.orm import Session from wrolpi import flags +from wrolpi.collections.models import Collection +from wrolpi.collections.sync import get_playlists_directory, MANAGED_NAME from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary, wrol_mode_enabled, \ get_config_by_file_name, SYNCED_CONFIG_FILE_NAMES from wrolpi.dates import now, is_within_daily_window +from wrolpi.db import get_db_session from wrolpi.errors import InvalidConfig from wrolpi.events import Events from wrolpi.sync.config import get_sync_config from wrolpi.sync.status import get_sync_status, update_sync_status +from wrolpi.tags import get_tags_directory from wrolpi.vars import DEFAULT_HTTP_HEADERS, PYTEST from wrolpi.version import __version__ @@ -402,10 +407,11 @@ class PrimaryClient: async def download_file(client: PrimaryClient, file: ManifestFile, destination: pathlib.Path, - on_bytes=None) -> int: + on_bytes=None, before_replace: Callable[[pathlib.Path], None] = None) -> int: """Download one file to `destination` with the primary's modification time. Returns the bytes received. - A partial file left by an earlier attempt is resumed.""" + A partial file left by an earlier attempt is resumed. `before_replace` is called with the complete new version + just before it takes the place of `destination`.""" partial = partial_path(destination) destination.parent.mkdir(parents=True, exist_ok=True) @@ -449,10 +455,95 @@ async def download_file(client: PrimaryClient, file: ManifestFile, destination: if (size := partial.stat().st_size) != file.size: raise SyncError(f'{file.path} is {size} bytes, the primary said {file.size}') os.utime(partial, (file.mtime, file.mtime)) + if before_replace: + before_replace(partial) os.replace(partial, destination) return received +def link_directories() -> List[tuple]: + """Where hard links of library files are kept, each as (directory, recursive): the Tags Directory, the + playlists directory, and the directory of every playlist that has its own.""" + directories = [(get_tags_directory(), True), (get_playlists_directory(), True)] + with get_db_session() as session: + custom = session.execute( + select(Collection.directory).where(Collection.kind == 'playlist', Collection.directory.isnot(None)) + ).scalars().all() + # A playlist's own directory may be any folder of the library; only the links directly in it are its own. + directories.extend((pathlib.Path(i), False) for i in custom) + return directories + + +def _is_link_name(link_name: str, file_name: str) -> bool: + """True if `link_name` is what the Tags Directory (the file's name) or a playlist (the name after its position) + calls a link to a file named `file_name`.""" + return link_name == file_name or bool(MANAGED_NAME.match(link_name)) and link_name[5:] == file_name + + +def find_links(files: List[pathlib.Path]) -> Dict[pathlib.Path, List[pathlib.Path]]: + """The Tags Directory and playlist links of each of `files`: the files in `link_directories` that are the same + inode as the file and are named for it. A file without other hard links has none and is left out.""" + inodes = dict() + for file in files: + try: + stat = file.lstat() + except FileNotFoundError: + continue + if stat.st_nlink > 1: + # Library files may be hard links of each other, so several files can share an inode. + inodes.setdefault((stat.st_dev, stat.st_ino), []).append(file) + links = dict() + if not inodes: + return links + + seen = set() + for directory, recursive in link_directories(): + roots = (i[0] for i in os.walk(directory)) if recursive else (str(directory),) + for root in roots: + if root in seen: + continue + seen.add(root) + try: + names = os.listdir(root) + except OSError: + continue + for name in names: + path = pathlib.Path(root, name) + try: + stat = path.lstat() + except OSError: + continue + for file in inodes.get((stat.st_dev, stat.st_ino), ()): + if _is_link_name(name, file.name): + links.setdefault(file, []).append(path) + return links + + +def move_links(links: List[pathlib.Path], old_version: pathlib.Path, new_version: pathlib.Path): + """Make each of `links` that is still a link to `old_version` a link to `new_version`. + + The Tags Directory and a playlist refuse to delete a file that has no other link, because it may be a last + copy. A link left on a replaced version is exactly that: once it is no longer wanted (the primary changed the + file's tags, or reordered the playlist) it would stay forever.""" + try: + old = old_version.lstat() + except FileNotFoundError: + return + for link in links: + # Short, so it fits wherever the link's own name does. + temporary = link.with_name(f'.{uuid.uuid4().hex}.sync') + try: + stat = link.lstat() + if (stat.st_dev, stat.st_ino) != (old.st_dev, old.st_ino): + continue + # Linked beside it, then renamed over it: the link is never missing. + os.link(new_version, temporary) + os.replace(temporary, link) + except OSError as e: + logger.error(f'Failed to move the link {repr(str(link))}: {e}') + temporary.unlink(missing_ok=True) + + # --------------------------------------------------------------------------------------------------------------------- # Secondary: configs. @@ -790,6 +881,9 @@ async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]: page_downloaded = list() if wanted and not disk_full: semaphore = asyncio.Semaphore(DOWNLOAD_CONCURRENCY) + # A file's links move to its new version before the new version takes its place, so a sync that is + # interrupted in between downloads nothing again and still leaves no link on the old version. + links = await asyncio.to_thread(find_links, [i for _, i in wanted]) async def download(file_: ManifestFile, destination_: pathlib.Path): nonlocal disk_full @@ -797,7 +891,11 @@ async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]: if disk_full: return None try: - await download_file(client, file_, destination_, count_bytes) + before_replace = None + if destination_ in links: + def before_replace(new_version: pathlib.Path): + move_links(links[destination_], destination_, new_version) + await download_file(client, file_, destination_, count_bytes, before_replace) downloaded.append(destination_) return destination_ except SyncDiskFull as e: diff --git a/wrolpi/sync/test/test_engine.py b/wrolpi/sync/test/test_engine.py index 218d2294..fc5f886c 100644 --- a/wrolpi/sync/test/test_engine.py +++ b/wrolpi/sync/test/test_engine.py @@ -5,6 +5,8 @@ from collections import namedtuple import pytest import yaml +from wrolpi.collections.models import Collection +from wrolpi.collections.sync import get_playlists_directory, _safe_unlink_managed from wrolpi.common import get_wrolpi_config, set_test_media_directory from wrolpi.db import get_db_session from wrolpi.errors import InvalidConfig @@ -13,7 +15,7 @@ from wrolpi.sync import lib from wrolpi.sync.config import get_sync_config from wrolpi.sync.conftest import manifest_page from wrolpi.sync.status import get_sync_status, update_sync_status -from wrolpi.tags import get_tags_config, Tag, TagFile +from wrolpi.tags import get_tags_config, Tag, TagFile, _delete_extra_tags_directory_paths @pytest.mark.parametrize('path, allowed', [ @@ -273,6 +275,171 @@ async def test_run_sync(async_client, test_session, primary, secondary_directory assert not [i for i in primary.state['requests'] if not i[0].startswith('/media/config/')] +@pytest.mark.asyncio +async def test_run_sync_replaced_file_keeps_its_links(async_client, primary, secondary_directory, make_secondary, + fake_primary_api): + """The Tags Directory and playlists links of a file the sync replaces are links to the new version. A link to + the old version would be the old version's last copy, which neither directory ever deletes.""" + changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) + same = write_primary_file(primary, 'docs/same.txt', b'unchanged') + fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime), (same.path, same.size, same.mtime))], + primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + # The secondary has an older version of one file, linked into a tag and a playlist. + notes = secondary_directory / 'docs/notes.txt' + unchanged = secondary_directory / 'docs/same.txt' + notes.parent.mkdir(parents=True) + notes.write_bytes(b'old notes') + unchanged.write_bytes(b'unchanged') + os.utime(unchanged, (same.mtime, same.mtime)) + tag_link = secondary_directory / 'tags/one/notes.txt' + playlist_link = get_playlists_directory() / 'Reading/0001_notes.txt' + unchanged_link = secondary_directory / 'tags/one/same.txt' + # The only copy of a file that someone put in the Tags Directory themselves. + stranger = secondary_directory / 'tags/one/stranger.txt' + for source, link in ((notes, tag_link), (notes, playlist_link), (unchanged, unchanged_link)): + link.parent.mkdir(parents=True, exist_ok=True) + link.hardlink_to(source) + stranger.write_bytes(b'old notes') + + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert status['files_downloaded'] == 1 + assert notes.read_bytes() == b'the new notes' + for link in (tag_link, playlist_link): + assert link.samefile(notes), f'{link} is still the old version' + assert notes.stat().st_nlink == 3 + assert unchanged_link.samefile(unchanged) + assert stranger.read_bytes() == b'old notes' and stranger.stat().st_nlink == 1 + assert not [i for i in secondary_directory.rglob('.*') if i.is_file()], 'no temporary file is left behind' + + # The links are no longer wanted (the tag was removed, the playlist reordered): both directories can delete them. + _delete_extra_tags_directory_paths(secondary_directory / 'tags', [unchanged_link, stranger]) + assert not tag_link.exists() + assert _safe_unlink_managed(playlist_link) + assert notes.read_bytes() == b'the new notes' and notes.stat().st_nlink == 1 + + +@pytest.mark.asyncio +async def test_run_sync_replaced_file_in_a_playlist_with_its_own_directory( + async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api): + """A playlist may keep its links in any folder of the library; a replaced file's link there is moved too, and + the folder's other files are left alone.""" + changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) + fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + notes = secondary_directory / 'docs/notes.txt' + notes.parent.mkdir(parents=True) + notes.write_bytes(b'old notes') + directory = secondary_directory / 'my lists/Reading' + (directory / 'deeper').mkdir(parents=True) + link = directory / '0001_notes.txt' + link.hardlink_to(notes) + # A hard link of the file that the playlist did not make: not its name, or not directly in its folder. + renamed = directory / 'my copy of the notes.txt' + deeper = directory / 'deeper/0001_notes.txt' + renamed.hardlink_to(notes) + deeper.hardlink_to(notes) + test_session.add(Collection(name='Reading', kind='playlist', directory=directory)) + test_session.commit() + + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert link.samefile(notes) and link.read_bytes() == b'the new notes' + assert renamed.read_bytes() == b'old notes' and deeper.read_bytes() == b'old notes' + assert _safe_unlink_managed(link) + + +@pytest.mark.asyncio +async def test_run_sync_replaced_file_with_a_hard_linked_twin(async_client, primary, secondary_directory, + make_secondary, fake_primary_api): + """Two library files may be hard links of each other. When one of them changes, only links named for it move; + its twin's link is still its twin.""" + changed = write_primary_file(primary, 'docs/a.txt', b'the new a', mtime=1_700_000_500.0) + fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + a, b = secondary_directory / 'docs/a.txt', secondary_directory / 'docs/b.txt' + a.parent.mkdir(parents=True) + a.write_bytes(b'shared') + b.hardlink_to(a) + a_link, b_link = secondary_directory / 'tags/one/a.txt', secondary_directory / 'tags/one/b.txt' + a_link.parent.mkdir(parents=True) + a_link.hardlink_to(a) + b_link.hardlink_to(b) + + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert a.read_bytes() == b'the new a' and a_link.samefile(a) + assert b.read_bytes() == b'shared' and b_link.samefile(b) + + +@pytest.mark.asyncio +async def test_run_sync_interrupted_before_a_file_is_replaced(async_client, primary, secondary_directory, + make_secondary, fake_primary_api, monkeypatch): + """A sync that stops between downloading a new version and putting it in place has left no link on the old + version once a later sync finishes the job, although that sync downloads nothing.""" + changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) + fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + notes = secondary_directory / 'docs/notes.txt' + notes.parent.mkdir(parents=True) + notes.write_bytes(b'old notes') + link = secondary_directory / 'tags/one/notes.txt' + link.parent.mkdir(parents=True) + link.hardlink_to(notes) + + replace = os.replace + + def power_cut(source, destination): + if pathlib.Path(destination) == notes: + raise OSError('the power went out') + return replace(source, destination) + + monkeypatch.setattr(lib.os, 'replace', power_cut) + status = await lib.run_sync() + assert status['files_failed'] == 1 + assert notes.read_bytes() == b'old notes', 'the file was not replaced' + monkeypatch.setattr(lib.os, 'replace', replace) + + primary.state['requests'].clear() + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert status['files_downloaded'] == 1 and status['bytes_downloaded'] == 0 + assert notes.read_bytes() == b'the new notes' + assert link.samefile(notes) and notes.stat().st_nlink == 2 + assert not [i for i in secondary_directory.rglob('.*') if i.is_file()] + + +def test_move_links_of_a_long_name(test_session, test_directory): + """A link whose name is as long as a file name may be is still moved.""" + name = 'n' * 240 + '.txt' + old, new = test_directory / 'docs' / name, test_directory / 'docs/new version' + old.parent.mkdir() + old.write_bytes(b'old') + new.write_bytes(b'new') + link = get_playlists_directory() / 'Reading' / f'0001_{name}' + link.parent.mkdir(parents=True) + link.hardlink_to(old) + + assert lib.find_links([old]) == {old: [link]} + lib.move_links([link], old, new) + + assert link.samefile(new) + assert [i.name for i in link.parent.iterdir()] == [link.name] + + @pytest.mark.asyncio async def test_run_sync_refuses_unsafe_paths(async_client, primary, secondary_directory, make_secondary, fake_primary_api):