A file the sync replaces keeps its Tags Directory and playlist links (#986)

* A file the sync replaces keeps its Tags Directory and playlist links

A secondary replaces a changed file with a new inode, so the file's hard links in the Tags Directory and the
playlists directory were left holding the old version.  Both directories repair a link that is still wanted,
but refuse to delete one with no other link.  When the primary also changed the file's tags, or reordered the
playlist, in the same sync, the old link was no longer wanted and could never be deleted.

After each page's downloads, every link to a replaced version in those two directories now becomes a link to
the new version.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>

* A replaced file's links move before the new version takes its place

Review of the first version found three gaps:

- A playlist with its own directory keeps its links outside the playlists directory; those were not looked at.
- The links were repaired after a whole page of downloads, so a sync that stopped mid-page left the files it had
  already replaced with their old links, and a later sync saw nothing to do.
- Two library files that are hard links of each other share an inode, so the link of the one that did not change
  was pointed at the one that did.

The links of a page's files are now found before its downloads, in the Tags Directory, the playlists directory
and each playlist's own directory, and only by the name those give a link.  Each file's links are moved to the
new version just before it is renamed into place.  The temporary name is short, so a long file name still fits.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-17 17:06:22 -07:00 • committed by GitHub
parent dc219e0775
commit d11b72bbd1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 270 additions and 5 deletions

View file

@ -26,10 +26,11 @@ import re
import shutil
import ssl
import time
import uuid
from dataclasses import dataclass
from datetime import datetime
from http import HTTPStatus
from typing import AsyncGenerator, List, Optional
from typing import AsyncGenerator, Callable, Dict, List, Optional
from urllib.parse import quote
import aiohttp
@ -38,13 +39,17 @@ from sqlalchemy import func, select
from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.collections.models import Collection
from wrolpi.collections.sync import get_playlists_directory, MANAGED_NAME
from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary, wrol_mode_enabled, \
get_config_by_file_name, SYNCED_CONFIG_FILE_NAMES
from wrolpi.dates import now, is_within_daily_window
from wrolpi.db import get_db_session
from wrolpi.errors import InvalidConfig
from wrolpi.events import Events
from wrolpi.sync.config import get_sync_config
from wrolpi.sync.status import get_sync_status, update_sync_status
from wrolpi.tags import get_tags_directory
from wrolpi.vars import DEFAULT_HTTP_HEADERS, PYTEST
from wrolpi.version import __version__
@ -402,10 +407,11 @@ class PrimaryClient:
async def download_file(client: PrimaryClient, file: ManifestFile, destination: pathlib.Path,
on_bytes=None) -> int:
on_bytes=None, before_replace: Callable[[pathlib.Path], None] = None) -> int:
"""Download one file to `destination` with the primary's modification time. Returns the bytes received.
A partial file left by an earlier attempt is resumed."""
A partial file left by an earlier attempt is resumed. `before_replace` is called with the complete new version
just before it takes the place of `destination`."""
partial = partial_path(destination)
destination.parent.mkdir(parents=True, exist_ok=True)
@ -449,10 +455,95 @@ async def download_file(client: PrimaryClient, file: ManifestFile, destination:
if (size := partial.stat().st_size) != file.size:
raise SyncError(f'{file.path} is {size} bytes, the primary said {file.size}')
os.utime(partial, (file.mtime, file.mtime))
if before_replace:
before_replace(partial)
os.replace(partial, destination)
return received
def link_directories() -> List[tuple]:
"""Where hard links of library files are kept, each as (directory, recursive): the Tags Directory, the
playlists directory, and the directory of every playlist that has its own."""
directories = [(get_tags_directory(), True), (get_playlists_directory(), True)]
with get_db_session() as session:
custom = session.execute(
select(Collection.directory).where(Collection.kind == 'playlist', Collection.directory.isnot(None))
).scalars().all()
# A playlist's own directory may be any folder of the library; only the links directly in it are its own.
directories.extend((pathlib.Path(i), False) for i in custom)
return directories
def _is_link_name(link_name: str, file_name: str) -> bool:
"""True if `link_name` is what the Tags Directory (the file's name) or a playlist (the name after its position)
calls a link to a file named `file_name`."""
return link_name == file_name or bool(MANAGED_NAME.match(link_name)) and link_name[5:] == file_name
def find_links(files: List[pathlib.Path]) -> Dict[pathlib.Path, List[pathlib.Path]]:
"""The Tags Directory and playlist links of each of `files`: the files in `link_directories` that are the same
inode as the file and are named for it. A file without other hard links has none and is left out."""
inodes = dict()
for file in files:
try:
stat = file.lstat()
except FileNotFoundError:
continue
if stat.st_nlink > 1:
# Library files may be hard links of each other, so several files can share an inode.
inodes.setdefault((stat.st_dev, stat.st_ino), []).append(file)
links = dict()
if not inodes:
return links
seen = set()
for directory, recursive in link_directories():
roots = (i[0] for i in os.walk(directory)) if recursive else (str(directory),)
for root in roots:
if root in seen:
continue
seen.add(root)
try:
names = os.listdir(root)
except OSError:
continue
for name in names:
path = pathlib.Path(root, name)
try:
stat = path.lstat()
except OSError:
continue
for file in inodes.get((stat.st_dev, stat.st_ino), ()):
if _is_link_name(name, file.name):
links.setdefault(file, []).append(path)
return links
def move_links(links: List[pathlib.Path], old_version: pathlib.Path, new_version: pathlib.Path):
"""Make each of `links` that is still a link to `old_version` a link to `new_version`.
The Tags Directory and a playlist refuse to delete a file that has no other link, because it may be a last
copy. A link left on a replaced version is exactly that: once it is no longer wanted (the primary changed the
file's tags, or reordered the playlist) it would stay forever."""
try:
old = old_version.lstat()
except FileNotFoundError:
return
for link in links:
# Short, so it fits wherever the link's own name does.
temporary = link.with_name(f'.{uuid.uuid4().hex}.sync')
try:
stat = link.lstat()
if (stat.st_dev, stat.st_ino) != (old.st_dev, old.st_ino):
continue
# Linked beside it, then renamed over it: the link is never missing.
os.link(new_version, temporary)
os.replace(temporary, link)
except OSError as e:
logger.error(f'Failed to move the link {repr(str(link))}: {e}')
temporary.unlink(missing_ok=True)
# ---------------------------------------------------------------------------------------------------------------------
# Secondary: configs.
@ -790,6 +881,9 @@ async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]:
page_downloaded = list()
if wanted and not disk_full:
semaphore = asyncio.Semaphore(DOWNLOAD_CONCURRENCY)
# A file's links move to its new version before the new version takes its place, so a sync that is
# interrupted in between downloads nothing again and still leaves no link on the old version.
links = await asyncio.to_thread(find_links, [i for _, i in wanted])
async def download(file_: ManifestFile, destination_: pathlib.Path):
nonlocal disk_full
@ -797,7 +891,11 @@ async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]:
if disk_full:
return None
try:
await download_file(client, file_, destination_, count_bytes)
before_replace = None
if destination_ in links:
def before_replace(new_version: pathlib.Path):
move_links(links[destination_], destination_, new_version)
await download_file(client, file_, destination_, count_bytes, before_replace)
downloaded.append(destination_)
return destination_
except SyncDiskFull as e:

View file

@ -5,6 +5,8 @@ from collections import namedtuple
import pytest
import yaml
from wrolpi.collections.models import Collection
from wrolpi.collections.sync import get_playlists_directory, _safe_unlink_managed
from wrolpi.common import get_wrolpi_config, set_test_media_directory
from wrolpi.db import get_db_session
from wrolpi.errors import InvalidConfig
@ -13,7 +15,7 @@ from wrolpi.sync import lib
from wrolpi.sync.config import get_sync_config
from wrolpi.sync.conftest import manifest_page
from wrolpi.sync.status import get_sync_status, update_sync_status
from wrolpi.tags import get_tags_config, Tag, TagFile
from wrolpi.tags import get_tags_config, Tag, TagFile, _delete_extra_tags_directory_paths
@pytest.mark.parametrize('path, allowed', [
@ -273,6 +275,171 @@ async def test_run_sync(async_client, test_session, primary, secondary_directory
assert not [i for i in primary.state['requests'] if not i[0].startswith('/media/config/')]
@pytest.mark.asyncio
async def test_run_sync_replaced_file_keeps_its_links(async_client, primary, secondary_directory, make_secondary,
fake_primary_api):
"""The Tags Directory and playlists links of a file the sync replaces are links to the new version. A link to
the old version would be the old version's last copy, which neither directory ever deletes."""
changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0)
same = write_primary_file(primary, 'docs/same.txt', b'unchanged')
fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime), (same.path, same.size, same.mtime))],
primary.directory)
set_test_media_directory(secondary_directory)
make_secondary(primary.url)
# The secondary has an older version of one file, linked into a tag and a playlist.
notes = secondary_directory / 'docs/notes.txt'
unchanged = secondary_directory / 'docs/same.txt'
notes.parent.mkdir(parents=True)
notes.write_bytes(b'old notes')
unchanged.write_bytes(b'unchanged')
os.utime(unchanged, (same.mtime, same.mtime))
tag_link = secondary_directory / 'tags/one/notes.txt'
playlist_link = get_playlists_directory() / 'Reading/0001_notes.txt'
unchanged_link = secondary_directory / 'tags/one/same.txt'
# The only copy of a file that someone put in the Tags Directory themselves.
stranger = secondary_directory / 'tags/one/stranger.txt'
for source, link in ((notes, tag_link), (notes, playlist_link), (unchanged, unchanged_link)):
link.parent.mkdir(parents=True, exist_ok=True)
link.hardlink_to(source)
stranger.write_bytes(b'old notes')
status = await lib.run_sync()
assert status['state'] == 'idle', status['last_error']
assert status['files_downloaded'] == 1
assert notes.read_bytes() == b'the new notes'
for link in (tag_link, playlist_link):
assert link.samefile(notes), f'{link} is still the old version'
assert notes.stat().st_nlink == 3
assert unchanged_link.samefile(unchanged)
assert stranger.read_bytes() == b'old notes' and stranger.stat().st_nlink == 1
assert not [i for i in secondary_directory.rglob('.*') if i.is_file()], 'no temporary file is left behind'
# The links are no longer wanted (the tag was removed, the playlist reordered): both directories can delete them.
_delete_extra_tags_directory_paths(secondary_directory / 'tags', [unchanged_link, stranger])
assert not tag_link.exists()
assert _safe_unlink_managed(playlist_link)
assert notes.read_bytes() == b'the new notes' and notes.stat().st_nlink == 1
@pytest.mark.asyncio
async def test_run_sync_replaced_file_in_a_playlist_with_its_own_directory(
async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api):
"""A playlist may keep its links in any folder of the library; a replaced file's link there is moved too, and
the folder's other files are left alone."""
changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0)
fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory)
set_test_media_directory(secondary_directory)
make_secondary(primary.url)
notes = secondary_directory / 'docs/notes.txt'
notes.parent.mkdir(parents=True)
notes.write_bytes(b'old notes')
directory = secondary_directory / 'my lists/Reading'
(directory / 'deeper').mkdir(parents=True)
link = directory / '0001_notes.txt'
link.hardlink_to(notes)
# A hard link of the file that the playlist did not make: not its name, or not directly in its folder.
renamed = directory / 'my copy of the notes.txt'
deeper = directory / 'deeper/0001_notes.txt'
renamed.hardlink_to(notes)
deeper.hardlink_to(notes)
test_session.add(Collection(name='Reading', kind='playlist', directory=directory))
test_session.commit()
status = await lib.run_sync()
assert status['state'] == 'idle', status['last_error']
assert link.samefile(notes) and link.read_bytes() == b'the new notes'
assert renamed.read_bytes() == b'old notes' and deeper.read_bytes() == b'old notes'
assert _safe_unlink_managed(link)
@pytest.mark.asyncio
async def test_run_sync_replaced_file_with_a_hard_linked_twin(async_client, primary, secondary_directory,
make_secondary, fake_primary_api):
"""Two library files may be hard links of each other. When one of them changes, only links named for it move;
its twin's link is still its twin."""
changed = write_primary_file(primary, 'docs/a.txt', b'the new a', mtime=1_700_000_500.0)
fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory)
set_test_media_directory(secondary_directory)
make_secondary(primary.url)
a, b = secondary_directory / 'docs/a.txt', secondary_directory / 'docs/b.txt'
a.parent.mkdir(parents=True)
a.write_bytes(b'shared')
b.hardlink_to(a)
a_link, b_link = secondary_directory / 'tags/one/a.txt', secondary_directory / 'tags/one/b.txt'
a_link.parent.mkdir(parents=True)
a_link.hardlink_to(a)
b_link.hardlink_to(b)
status = await lib.run_sync()
assert status['state'] == 'idle', status['last_error']
assert a.read_bytes() == b'the new a' and a_link.samefile(a)
assert b.read_bytes() == b'shared' and b_link.samefile(b)
@pytest.mark.asyncio
async def test_run_sync_interrupted_before_a_file_is_replaced(async_client, primary, secondary_directory,
make_secondary, fake_primary_api, monkeypatch):
"""A sync that stops between downloading a new version and putting it in place has left no link on the old
version once a later sync finishes the job, although that sync downloads nothing."""
changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0)
fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory)
set_test_media_directory(secondary_directory)
make_secondary(primary.url)
notes = secondary_directory / 'docs/notes.txt'
notes.parent.mkdir(parents=True)
notes.write_bytes(b'old notes')
link = secondary_directory / 'tags/one/notes.txt'
link.parent.mkdir(parents=True)
link.hardlink_to(notes)
replace = os.replace
def power_cut(source, destination):
if pathlib.Path(destination) == notes:
raise OSError('the power went out')
return replace(source, destination)
monkeypatch.setattr(lib.os, 'replace', power_cut)
status = await lib.run_sync()
assert status['files_failed'] == 1
assert notes.read_bytes() == b'old notes', 'the file was not replaced'
monkeypatch.setattr(lib.os, 'replace', replace)
primary.state['requests'].clear()
status = await lib.run_sync()
assert status['state'] == 'idle', status['last_error']
assert status['files_downloaded'] == 1 and status['bytes_downloaded'] == 0
assert notes.read_bytes() == b'the new notes'
assert link.samefile(notes) and notes.stat().st_nlink == 2
assert not [i for i in secondary_directory.rglob('.*') if i.is_file()]
def test_move_links_of_a_long_name(test_session, test_directory):
"""A link whose name is as long as a file name may be is still moved."""
name = 'n' * 240 + '.txt'
old, new = test_directory / 'docs' / name, test_directory / 'docs/new version'
old.parent.mkdir()
old.write_bytes(b'old')
new.write_bytes(b'new')
link = get_playlists_directory() / 'Reading' / f'0001_{name}'
link.parent.mkdir(parents=True)
link.hardlink_to(old)
assert lib.find_links([old]) == {old: [link]}
lib.move_links([link], old, new)
assert link.samefile(new)
assert [i.name for i in link.parent.iterdir()] == [link.name]
@pytest.mark.asyncio
async def test_run_sync_refuses_unsafe_paths(async_client, primary, secondary_directory, make_secondary,
fake_primary_api):