mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
The old sync engine is removed (#998)
* The old sync engine is removed Part 4 of replacing the sync. Since the previous part a secondary copies its primary with rsync, run by the Controller, so nothing used the HTTP engine any more: - the manifest (/api/sync/manifest), the download of a FileGroup's files over HTTP, the copy and import of each synced config, the moving of a replaced file's links, the request, cancel and tick of the old sync; - the API lock that answered 503 SYNCING while that engine ran, its Event, the sync_running flag, the old status record in /api/status and its schema, and the frontend's handling of 503 SYNCING; - their tests. What a primary says about itself (/api/sync/info), the secondary's settings (its primary, the folders it does not copy, Test Connection, the folder chooser), taking the primary's library settings when the API starts, and the daily window of the backup drive stay. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * The primary's sync info no longer reports the largest FileGroup id Only the old engine read it. Review also found a docstring that still spoke of a manifest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * A sync is announced again Since the Controller runs a sync, nothing sent its events: the old engine did, and the Controller cannot send the API's events (the API is stopped for most of a sync). The API's loop now reads the Controller's record of the last sync and sends one event per finished sync: "Synced N files from the primary" (with the files moved to the trash, or a warning) or "Sync from the primary failed". As before, a sync that changed nothing is not announced (within the window that is every 15 minutes), nor one that was stopped on purpose. What was announced is kept in config/.sync, which a sync never touches, so the restart every sync makes neither repeats nor loses one. Found by review, and by the dead-code check, which saw the two event methods unused. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8cfe36b3ea
commit
38885990f6
23 changed files with 354 additions and 1797 deletions
|
|
@ -34,11 +34,6 @@ export class ApiDownError extends Error {
|
|||
|
||||
const apiDownError = new ApiDownError('API is down');
|
||||
|
||||
// A secondary WROLPi refuses requests while it copies its library from its primary. Like a down API, callers
|
||||
// treat it quietly; the app shows the sync's progress instead (see SyncingOverlay).
|
||||
export class SyncingError extends ApiDownError {
|
||||
}
|
||||
|
||||
async function apiCall(url, method, body, ms = 60_000) {
|
||||
let init = {method};
|
||||
if (body !== undefined) {
|
||||
|
|
@ -64,9 +59,6 @@ async function apiCall(url, method, body, ms = 60_000) {
|
|||
let copy = response.clone();
|
||||
const content = await copy.json();
|
||||
const code = content['code'];
|
||||
if (response.status === 503 && code === 'SYNCING') {
|
||||
throw new SyncingError('This WROLPi is syncing from its primary');
|
||||
}
|
||||
console.error('API error response json', content);
|
||||
if (response.status === 403 && code === 'WROL_MODE_ENABLED') {
|
||||
toast({
|
||||
|
|
|
|||
|
|
@ -96,33 +96,6 @@ export const settingsFixture = (overrides = {}) => ({
|
|||
...overrides,
|
||||
});
|
||||
|
||||
/** The `sync` of GET /api/status: a secondary that has never synced. */
|
||||
export const syncStatusFixture = (overrides = {}) => ({
|
||||
state: 'idle',
|
||||
phase: null,
|
||||
requested: false,
|
||||
force: false,
|
||||
cancel_requested: false,
|
||||
started_at: null,
|
||||
finished_at: null,
|
||||
last_success_at: null,
|
||||
last_error: null,
|
||||
last_skip_reason: null,
|
||||
primary_url: null,
|
||||
primary_version: null,
|
||||
groups_seen: 0,
|
||||
max_id: 0,
|
||||
files_checked: 0,
|
||||
files_to_download: 0,
|
||||
files_downloaded: 0,
|
||||
bytes_downloaded: 0,
|
||||
files_failed: 0,
|
||||
files_excluded: 0,
|
||||
configs_imported: [],
|
||||
disk_full: false,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
/** GET /controller/api/sync/status: a secondary's sync that has never run (controller/lib/sync.py). */
|
||||
export const controllerSyncStatusFixture = (overrides = {}) => ({
|
||||
state: 'idle',
|
||||
|
|
@ -198,8 +171,6 @@ export const statusFixture = (overrides = {}) => ({
|
|||
memory_stats: {total: 4000000000, used: 1000000000, free: 3000000000, cached: 500000000},
|
||||
processes_stats: [],
|
||||
sanic_workers: {},
|
||||
// A secondary's sync (wrolpi.sync); idle on a primary.
|
||||
sync: syncStatusFixture(),
|
||||
// A backup to a second drive (wrolpi.sync.backup); idle without one.
|
||||
backup: backupStatusFixture(),
|
||||
version: '1.0.0',
|
||||
|
|
|
|||
|
|
@ -2183,10 +2183,9 @@ def enable_wrol_mode():
|
|||
get_wrolpi_config().wrol_mode = True
|
||||
from wrolpi.downloader import download_manager
|
||||
download_manager.stop()
|
||||
# A secondary in WROL Mode is in use; a running sync stops between pages. A backup stops too.
|
||||
from wrolpi.sync.lib import request_sync_cancel
|
||||
# A backup stops. (A secondary's sync is the Controller's, which stops at its next step when it finds WROL Mode
|
||||
# enabled; its API is stopped while it copies, so WROL Mode cannot be enabled then.)
|
||||
from wrolpi.sync.backup import request_backup_cancel
|
||||
request_sync_cancel()
|
||||
request_backup_cancel()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import queue
|
|||
from sanic import Sanic
|
||||
|
||||
from wrolpi.common import logger
|
||||
from wrolpi.sync.status import default_sync_status, default_backup_status
|
||||
from wrolpi.sync.status import default_backup_status
|
||||
from wrolpi.vars import LOG_LEVEL_INT
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -68,9 +68,6 @@ def attach_shared_contexts(app: Sanic):
|
|||
app.shared_ctx.download_manager_stopped.set()
|
||||
|
||||
app.shared_ctx.single_tasks_started = multiprocessing.Event()
|
||||
# Set while a secondary syncs (wrolpi.sync); the API refuses requests. An Event so the request middleware can
|
||||
# check it without a manager round trip.
|
||||
app.shared_ctx.sync_api_locked = multiprocessing.Event()
|
||||
app.shared_ctx.flags_initialized = multiprocessing.Event()
|
||||
|
||||
# Transcode
|
||||
|
|
@ -158,7 +155,6 @@ def reset_shared_contexts(app: Sanic):
|
|||
current_commit=None,
|
||||
commits_behind=0,
|
||||
git_branch=None,
|
||||
sync=default_sync_status(),
|
||||
backup=default_backup_status(),
|
||||
))
|
||||
app.shared_ctx.cache.clear()
|
||||
|
|
@ -248,7 +244,6 @@ def reset_shared_contexts(app: Sanic):
|
|||
# Clear events_history in place to preserve references held by fixtures
|
||||
del app.shared_ctx.events_history[:]
|
||||
app.shared_ctx.single_tasks_started.clear()
|
||||
app.shared_ctx.sync_api_locked.clear()
|
||||
app.shared_ctx.flags_initialized.clear()
|
||||
if hasattr(app.shared_ctx, 'flag_holders'):
|
||||
app.shared_ctx.flag_holders.clear()
|
||||
|
|
|
|||
|
|
@ -45,12 +45,6 @@ class WROLModeEnabled(APIError):
|
|||
status_code = HTTPStatus.FORBIDDEN
|
||||
|
||||
|
||||
class SyncInProgress(APIError):
|
||||
code = 'SYNCING'
|
||||
summary = 'This WROLPi is copying its library from its primary. Try again when the sync finishes.'
|
||||
status_code = HTTPStatus.SERVICE_UNAVAILABLE
|
||||
|
||||
|
||||
class SecondaryRole(APIError):
|
||||
code = 'SECONDARY_ROLE'
|
||||
summary = 'This method is disabled on a secondary WROLPi.'
|
||||
|
|
|
|||
|
|
@ -161,8 +161,6 @@ map_search_building = Flag('map_search_building')
|
|||
# The global refresh has been performed. This is False on a fresh instance of WROLPi.
|
||||
refresh_complete = Flag('refresh_complete', store_db=True)
|
||||
|
||||
# A secondary WROLPi is copying its library from its primary (see wrolpi.sync). Its API refuses requests meanwhile.
|
||||
sync_running = Flag('sync_running')
|
||||
# This WROLPi is copying its media directory to its backup drive (see wrolpi.sync.backup).
|
||||
backup_running = Flag('backup_running')
|
||||
|
||||
|
|
@ -190,7 +188,6 @@ def get_flags() -> dict:
|
|||
media_mounted=media_mounted.is_set(),
|
||||
outdated_zims=outdated_zims.is_set(),
|
||||
refresh_complete=refresh_complete.is_set(),
|
||||
sync_running=sync_running.is_set(),
|
||||
)
|
||||
return flags
|
||||
|
||||
|
|
|
|||
|
|
@ -55,8 +55,7 @@ SHARED_LOCK_HEAL_TIMEOUT = 10
|
|||
# when the recorded holder pid is dead, because a live worker may be inside the block right now.
|
||||
# file_worker_busy is deliberately absent: the collection-move path in an API worker holds it as a mutex, and
|
||||
# process_queue already drops a stale set on its next idle tick.
|
||||
PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active', 'sync_running',
|
||||
'backup_running')
|
||||
PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active', 'backup_running')
|
||||
SHARED_PROGRESS_FLAGS = ('file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup',
|
||||
'map_search_building')
|
||||
|
||||
|
|
|
|||
|
|
@ -141,7 +141,6 @@ class FlagsStatusResponse:
|
|||
media_mounted: bool
|
||||
outdated_zims: bool
|
||||
refresh_complete: bool
|
||||
sync_running: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -201,33 +200,6 @@ class DownloadsSummaryResponse:
|
|||
daily_limit_reached: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncStatusResponse:
|
||||
"""The progress of a secondary WROLPi copying its library from its primary (see wrolpi.sync)."""
|
||||
state: str # 'idle', 'running', or 'error'
|
||||
phase: Optional[str] # 'files', 'configs', or None
|
||||
requested: bool
|
||||
force: bool
|
||||
cancel_requested: bool
|
||||
started_at: Optional[str]
|
||||
finished_at: Optional[str]
|
||||
last_success_at: Optional[str]
|
||||
last_error: Optional[str]
|
||||
last_skip_reason: Optional[str]
|
||||
primary_url: Optional[str]
|
||||
primary_version: Optional[str]
|
||||
groups_seen: int
|
||||
max_id: int
|
||||
files_checked: int
|
||||
files_to_download: int
|
||||
files_downloaded: int
|
||||
bytes_downloaded: int
|
||||
files_failed: int
|
||||
files_excluded: int
|
||||
configs_imported: List[str]
|
||||
disk_full: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupStatusResponse:
|
||||
"""The progress of copying this WROLPi's media directory to its backup drive (see wrolpi.sync.backup)."""
|
||||
|
|
@ -277,7 +249,6 @@ class StatusResponse:
|
|||
memory_stats: MemoryStatusStat
|
||||
processes_stats: List[ProcessStatusStat]
|
||||
sanic_workers: dict
|
||||
sync: SyncStatusResponse
|
||||
version: str
|
||||
# `str` here was wrong too; wrol_mode_enabled() returns a bool.
|
||||
wrol_mode: bool
|
||||
|
|
|
|||
|
|
@ -4,54 +4,30 @@ import re
|
|||
from http import HTTPStatus
|
||||
|
||||
from sanic import Request, Blueprint, response
|
||||
from sanic_ext import validate
|
||||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app, json_response, perpetual_signal
|
||||
from wrolpi.common import logger, get_wrolpi_config, is_secondary, wrol_mode_check, wrol_mode_enabled, \
|
||||
from wrolpi.api_utils import json_response, perpetual_signal
|
||||
from wrolpi.common import logger, is_secondary, wrol_mode_check, wrol_mode_enabled, \
|
||||
CONTROLLER_URL, aiohttp_post
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import InvalidConfig, SecondaryRole, SyncInProgress, WROLModeEnabled
|
||||
from wrolpi.sync import backup, lib, schema, snapshot
|
||||
from wrolpi.errors import InvalidConfig, SecondaryRole, WROLModeEnabled
|
||||
from wrolpi.sync import backup, lib, notify, schema, snapshot
|
||||
from wrolpi.sync.config import get_sync_config
|
||||
from wrolpi.sync.status import get_backup_status, get_snapshot_status, get_sync_status
|
||||
from wrolpi.sync.status import get_backup_status, get_snapshot_status
|
||||
from wrolpi.vars import DOCKERIZED
|
||||
|
||||
sync_bp = Blueprint('Sync', '/api/sync')
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
# Requests a secondary answers while it syncs: enough for the UI to show the sync's progress, and to cancel it.
|
||||
SYNC_ALLOWED_REQUESTS = frozenset({
|
||||
('GET', '/api/echo'),
|
||||
('GET', '/api/events/feed'),
|
||||
('GET', '/api/settings'),
|
||||
('GET', '/api/status'),
|
||||
('GET', '/api/sync/info'),
|
||||
('GET', '/api/sync/status'),
|
||||
('POST', '/api/sync/cancel'),
|
||||
})
|
||||
|
||||
|
||||
@api_app.middleware('request')
|
||||
async def check_sync_api_lock(request: Request):
|
||||
"""A secondary refuses requests while it replaces its library with its primary's (see wrolpi.sync)."""
|
||||
# The Event costs nothing to check; the Flag confirms it (the Event survives a perpetual process that died).
|
||||
if not api_app.shared_ctx.sync_api_locked.is_set() or not flags.sync_running.is_set():
|
||||
return
|
||||
if request.method == 'OPTIONS' or (request.method, request.path.rstrip('/')) in SYNC_ALLOWED_REQUESTS:
|
||||
return
|
||||
# Answered here rather than raised: the UI polls while it waits, and each refusal would be logged as an error.
|
||||
return json_response(dict(error=SyncInProgress.summary, code=SyncInProgress.code),
|
||||
SyncInProgress.status_code)
|
||||
|
||||
|
||||
@perpetual_signal(sleep=5)
|
||||
async def perpetual_sync_worker():
|
||||
# A secondary's sync is the Controller's (controller/lib/sync.py), which stops this API while it runs.
|
||||
await backup.backup_tick()
|
||||
await snapshot.snapshot_tick()
|
||||
# The Controller cannot send this API's events; announce the syncs it finished.
|
||||
notify.notify_tick()
|
||||
|
||||
|
||||
@sync_bp.get('/info')
|
||||
|
|
@ -64,22 +40,11 @@ async def get_info(_: Request):
|
|||
return json_response(await asyncio.to_thread(info))
|
||||
|
||||
|
||||
@sync_bp.get('/manifest')
|
||||
@openapi.definition(summary="One page of this WROLPi's files, for a secondary to compare with its own")
|
||||
@validate(query=schema.ManifestRequest)
|
||||
async def get_manifest(_: Request, query: schema.ManifestRequest):
|
||||
def page():
|
||||
with get_db_session() as session:
|
||||
return lib.build_manifest_page(session, query.after_id, query.limit)
|
||||
|
||||
return json_response(await asyncio.to_thread(page))
|
||||
|
||||
|
||||
@sync_bp.get('/status')
|
||||
@openapi.definition(summary="The progress of this secondary's sync")
|
||||
async def get_status(_: Request):
|
||||
return json_response(dict(sync=get_sync_status(), sync_running=bool(flags.sync_running.is_set()),
|
||||
backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set()),
|
||||
# A secondary's sync is the Controller's: /controller/api/sync/status.
|
||||
return json_response(dict(backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set()),
|
||||
snapshot=get_snapshot_status()))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from wrolpi.sync.status import get_backup_status, update_backup_status
|
|||
@dataclass
|
||||
class SyncConfigValidator:
|
||||
version: int = None
|
||||
# 'primary' or 'secondary'; a secondary copies its library from `primary_url` (see wrolpi.sync.lib).
|
||||
# 'primary' or 'secondary'; a secondary copies its library from `primary_url` (see controller/lib/sync.py).
|
||||
role: str = None
|
||||
primary_url: str = None
|
||||
# The daily window ('HH:MM') in which a secondary syncs; none means only "Sync Now".
|
||||
|
|
|
|||
|
|
@ -7,12 +7,11 @@ import os
|
|||
import pathlib
|
||||
import tempfile
|
||||
import threading
|
||||
from typing import Callable, List
|
||||
from typing import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.common import get_wrolpi_config, set_test_media_directory
|
||||
from wrolpi.sync import lib
|
||||
from wrolpi.common import set_test_media_directory
|
||||
from wrolpi.sync.config import get_sync_config
|
||||
|
||||
|
||||
|
|
@ -125,33 +124,6 @@ def make_secondary(test_wrolpi_config):
|
|||
return make
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_primary_api(monkeypatch):
|
||||
"""Answer the primary's sync API (info and manifest) from pages captured on the "primary" beforehand; media still
|
||||
comes from the `primary_media_server`."""
|
||||
|
||||
def fake(pages: List[dict], media_directory: pathlib.Path, role: str = 'primary', version: str = None):
|
||||
async def get_info(self):
|
||||
from wrolpi.version import __version__
|
||||
return dict(role=role, version=version or __version__, media_directory=str(media_directory),
|
||||
max_file_group_id=pages[-1]['max_id'] if pages else 0)
|
||||
|
||||
async def iter_manifest(self, limit: int = lib.MANIFEST_PAGE_LIMIT):
|
||||
for page in pages:
|
||||
yield page
|
||||
|
||||
monkeypatch.setattr(lib.PrimaryClient, 'get_info', get_info)
|
||||
monkeypatch.setattr(lib.PrimaryClient, 'iter_manifest', iter_manifest)
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
def manifest_page(*files: tuple, next_after_id=None, max_id: int = 1) -> dict:
|
||||
"""A manifest page of one group holding `files`, each (path, size, mtime)."""
|
||||
return dict(groups=[dict(id=1, paths=[dict(path=p, size=s, mtime=m) for p, s, m in files])],
|
||||
next_after_id=next_after_id, max_id=max_id, limit=lib.MANIFEST_PAGE_LIMIT)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def media_directory(path: pathlib.Path):
|
||||
set_test_media_directory(path)
|
||||
|
|
|
|||
|
|
@ -1,86 +1,54 @@
|
|||
"""Copy a primary WROLPi's library to a secondary WROLPi.
|
||||
"""The API's side of a secondary WROLPi copying its primary.
|
||||
|
||||
A secondary pulls; the primary only answers reads. The library is the media files plus the config YAMLs, because the
|
||||
database is rebuilt from both, so a sync is:
|
||||
The Controller runs a sync (controller/lib/sync.py): it stops this API, copies the primary's whole media directory
|
||||
with rsync from the primary's daemon (controller/rsyncd.conf), puts the primary's database snapshot
|
||||
(wrolpi.sync.snapshot) in place, and starts this API again. What is left here:
|
||||
|
||||
1. Walk the primary's manifest (`/api/sync/manifest`): every file of every FileGroup, paged by FileGroup id.
|
||||
2. Download each file the secondary lacks, or has with a different size or modification time, from the primary's
|
||||
`/media/` (Caddy). A file is written to a hidden partial (refresh never indexes hidden files), then renamed into
|
||||
place with the primary's modification time, so a later comparison is exact and an interrupted download resumes.
|
||||
3. Refresh the downloaded files, a page at a time.
|
||||
4. Copy the synced configs (`SYNCED_CONFIG_FILE_NAMES`) and import them. Files come first so the configs never
|
||||
refer to files the secondary does not have yet.
|
||||
|
||||
Files deleted on the primary are never deleted on the secondary.
|
||||
|
||||
While a sync runs the secondary's API refuses requests (`check_sync_api_lock`), so nobody edits a library that is
|
||||
being replaced.
|
||||
- what a primary tells a secondary about itself (`get_sync_info`),
|
||||
- the secondary's settings: its primary's address and the folders it does not copy,
|
||||
- the primary's library settings, which a secondary's API takes from the primary's wrolpi.yaml when it starts
|
||||
(`apply_primary_wrolpi_config`),
|
||||
- the daily window, which the backup drive uses too.
|
||||
"""
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import posixpath
|
||||
import re
|
||||
import shutil
|
||||
import ssl
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from http import HTTPStatus
|
||||
from typing import AsyncGenerator, Callable, Dict, List, Optional
|
||||
from urllib.parse import quote
|
||||
from typing import List, Optional
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import aiohttp
|
||||
import yaml
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi import flags
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.collections.sync import get_playlists_directory, MANAGED_NAME
|
||||
from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary, wrol_mode_enabled, \
|
||||
get_config_by_file_name, SYNCED_CONFIG_FILE_NAMES
|
||||
from wrolpi.dates import now, is_within_daily_window
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary
|
||||
from wrolpi.dates import is_within_daily_window
|
||||
from wrolpi.errors import InvalidConfig
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.sync.config import get_sync_config
|
||||
from wrolpi.sync.status import get_sync_status, update_sync_status
|
||||
from wrolpi.tags import get_tags_directory
|
||||
from wrolpi.vars import DEFAULT_HTTP_HEADERS, PYTEST
|
||||
from wrolpi.vars import DEFAULT_HTTP_HEADERS
|
||||
from wrolpi.version import __version__
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
MANIFEST_PAGE_LIMIT = 500
|
||||
# Files downloaded at once. Small files are dominated by round trips; four keeps a Pi's disk and CPU busy.
|
||||
DOWNLOAD_CONCURRENCY = 4
|
||||
DOWNLOAD_CHUNK_SIZE = 1024 * 1024
|
||||
# Never fill the secondary's drive; leave this much free.
|
||||
MIN_FREE_BYTES = 1024 ** 3
|
||||
# exFAT stores modification times in 2-second steps.
|
||||
MTIME_TOLERANCE_SECONDS = 2.0
|
||||
# How often a running sync reports its progress.
|
||||
# How often a running backup reports its progress.
|
||||
PROGRESS_INTERVAL_SECONDS = 2.0
|
||||
# Within the sync window, sync again this long after the last sync finished (an idle sync only compares sizes).
|
||||
SYNC_WINDOW_REPEAT_SECONDS = 15 * 60
|
||||
REFRESH_TIMEOUT_SECONDS = 60 * 60
|
||||
# Configs and directory listings are small; never read more than this from the primary.
|
||||
MAX_CONFIG_BYTES = 64 * 1024 * 1024
|
||||
# Answers and directory listings are small; never read more than this from the primary.
|
||||
MAX_RESPONSE_BYTES = 64 * 1024 * 1024
|
||||
READ_CHUNK_SIZE = 1024 * 1024
|
||||
# Enough for any real library; keeps a pasted list from growing without bound.
|
||||
MAX_EXCLUDED_DIRECTORIES = 500
|
||||
|
||||
# Top-level directories never written by a sync: the configs and database, and the Tags Directory (hard links that
|
||||
# the secondary rebuilds from tags.yaml).
|
||||
# Top-level folders that cannot be excluded: the configs and database, and the Tags Directory (hard links that the
|
||||
# secondary rebuilds).
|
||||
PROTECTED_DIRECTORIES = frozenset({'config', 'tags'})
|
||||
# Top-level folders never offered for exclusion: the protected ones, and the root-only folder of every ext4 drive.
|
||||
UNLISTED_DIRECTORIES = PROTECTED_DIRECTORIES | {'lost+found'}
|
||||
# Files that are still being downloaded on the primary: aria2 and yt-dlp partials, and yt-dlp format files.
|
||||
INCOMPLETE_FILE = re.compile(r'(\.part|\.aria2|\.ytdl|\.temp)$|\.f\d{2,4}\.[^.]+$')
|
||||
|
||||
# wrolpi.yaml keys that describe the library rather than this machine (hotspot, timezone, etc. stay local; the role
|
||||
# and other sync settings are in sync.yaml, which is never synced).
|
||||
|
|
@ -103,57 +71,8 @@ class SyncError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class SyncDiskFull(SyncError):
|
||||
pass
|
||||
|
||||
|
||||
class SyncCancelled(SyncError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ManifestFile:
|
||||
path: str # Relative to the media directory, '/' separated.
|
||||
size: int
|
||||
mtime: float
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
# Primary: the manifest.
|
||||
|
||||
|
||||
def build_manifest_page(session: Session, after_id: int = 0, limit: int = MANIFEST_PAGE_LIMIT) -> dict:
|
||||
"""One page of the primary's library: the files of the FileGroups after `after_id`, in id order.
|
||||
|
||||
Sizes and modification times come from the files themselves, not the database, which does not record them for
|
||||
every file of a group."""
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
media_directory = get_media_directory().resolve()
|
||||
rows = session.execute(
|
||||
select(FileGroup.id, FileGroup.directory, FileGroup.files)
|
||||
.where(FileGroup.id > after_id)
|
||||
.order_by(FileGroup.id)
|
||||
.limit(limit)
|
||||
).all()
|
||||
max_id = session.execute(select(func.max(FileGroup.id))).scalar() or 0
|
||||
|
||||
groups = list()
|
||||
for id_, directory, files in rows:
|
||||
paths = list()
|
||||
for entry in files or []:
|
||||
if not isinstance(entry, dict) or not entry.get('path'):
|
||||
continue
|
||||
path = pathlib.Path(entry['path'])
|
||||
if not path.is_absolute():
|
||||
# Files are stored relative to their FileGroup's directory; older rows stored absolute paths.
|
||||
path = pathlib.Path(directory) / path
|
||||
if manifest_file := _manifest_file(media_directory, path):
|
||||
paths.append(manifest_file)
|
||||
groups.append(dict(id=id_, paths=[dict(path=i.path, size=i.size, mtime=i.mtime) for i in paths]))
|
||||
|
||||
next_after_id = rows[-1][0] if len(rows) == limit else None
|
||||
return dict(groups=groups, next_after_id=next_after_id, max_id=max_id, limit=limit)
|
||||
# Primary: what it tells a secondary.
|
||||
|
||||
|
||||
def _is_protected(name: str) -> bool:
|
||||
|
|
@ -166,32 +85,13 @@ def _is_unlisted(name: str) -> bool:
|
|||
return name.casefold() in UNLISTED_DIRECTORIES
|
||||
|
||||
|
||||
def _manifest_file(media_directory: pathlib.Path, path: pathlib.Path) -> Optional[ManifestFile]:
|
||||
try:
|
||||
relative = path.relative_to(media_directory)
|
||||
except ValueError:
|
||||
return None
|
||||
parts = relative.parts
|
||||
if not parts or any(i.startswith('.') for i in parts) or _is_protected(parts[0]):
|
||||
return None
|
||||
if INCOMPLETE_FILE.search(path.name):
|
||||
return None
|
||||
try:
|
||||
stat = path.stat()
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return None
|
||||
if not path.is_file():
|
||||
return None
|
||||
return ManifestFile(relative.as_posix(), stat.st_size, stat.st_mtime)
|
||||
|
||||
|
||||
def get_sync_info(session: Session) -> dict:
|
||||
from wrolpi.files.models import FileGroup
|
||||
"""What a secondary's Controller checks before it replaces its library and database with this WROLPi's."""
|
||||
return dict(
|
||||
version=__version__.strip(),
|
||||
role=get_sync_config().role,
|
||||
# The database names files by their full path.
|
||||
media_directory=str(get_media_directory()),
|
||||
max_file_group_id=session.execute(select(func.max(FileGroup.id))).scalar() or 0,
|
||||
# A secondary that replaces its database with this one must run code of the same migration.
|
||||
database_revision=_database_revision(session),
|
||||
# An unmounted media directory looks like an empty library.
|
||||
|
|
@ -207,12 +107,12 @@ def _database_revision(session: Session) -> Optional[str]:
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
# Secondary: comparing and writing files.
|
||||
# Secondary: its primary, and the folders it does not copy.
|
||||
|
||||
|
||||
def resolve_destination(media_directory: pathlib.Path, path: str) -> Optional[pathlib.Path]:
|
||||
"""The secondary's path for a manifest path, or None if the path must not be written: outside the media directory,
|
||||
hidden, protected, or reached through a symlink."""
|
||||
"""The path in this WROLPi's media directory for `path` (relative, '/'-separated), or None if it may not be used:
|
||||
outside the media directory, hidden, protected, or reached through a symlink."""
|
||||
if not path or path.startswith('/') or '\\' in path:
|
||||
return None
|
||||
parts = pathlib.PurePosixPath(path).parts
|
||||
|
|
@ -290,45 +190,17 @@ async def list_primary_directories(path: str) -> List[dict]:
|
|||
return sorted(directories, key=lambda i: i['name'].lower())
|
||||
|
||||
|
||||
def needs_download(destination: pathlib.Path, file: ManifestFile) -> bool:
|
||||
try:
|
||||
stat = destination.stat()
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
return stat.st_size != file.size or abs(stat.st_mtime - file.mtime) > MTIME_TOLERANCE_SECONDS
|
||||
|
||||
|
||||
def partial_path(destination: pathlib.Path) -> pathlib.Path:
|
||||
"""Hidden, so a refresh never indexes a file that is still downloading."""
|
||||
return destination.with_name(f'.{destination.name}.partial')
|
||||
|
||||
|
||||
def _cpu_lacks_aes() -> bool:
|
||||
"""True when the CPU has no AES instructions (a Raspberry Pi 4). ChaCha20 is then about twice as fast."""
|
||||
try:
|
||||
cpuinfo = pathlib.Path('/proc/cpuinfo').read_text()
|
||||
except OSError:
|
||||
return False
|
||||
features = [i for i in cpuinfo.splitlines() if i.lower().startswith(('features', 'flags'))]
|
||||
return bool(features) and not any(' aes' in f' {i.split(":", 1)[-1]} ' for i in features)
|
||||
|
||||
|
||||
def make_sync_ssl_context() -> ssl.SSLContext:
|
||||
"""WROLPi serves HTTPS with its own certificate authority; on a trusted LAN (see the Bookshelf threat model) the
|
||||
primary is not verified. Pinning it would load the primary's `/ca.crt` here with `load_verify_locations`."""
|
||||
primary is not verified."""
|
||||
context = ssl.create_default_context()
|
||||
context.check_hostname = False
|
||||
context.verify_mode = ssl.CERT_NONE
|
||||
if _cpu_lacks_aes():
|
||||
# Python cannot order TLS 1.3 ciphers, and TLS 1.3 picks AES-GCM, which a Pi 4 decrypts at half the speed.
|
||||
context.maximum_version = ssl.TLSVersion.TLSv1_2
|
||||
context.set_ciphers('ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE+AESGCM')
|
||||
return context
|
||||
|
||||
|
||||
def normalize_primary_url(url: str) -> str:
|
||||
"""'https://host[:port]' with no path; raises ValueError otherwise."""
|
||||
from urllib.parse import urlparse
|
||||
url = (url or '').strip().rstrip('/')
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ('http', 'https') or not parsed.hostname:
|
||||
|
|
@ -343,15 +215,16 @@ def normalize_primary_url(url: str) -> str:
|
|||
|
||||
async def _read_limited(response: aiohttp.ClientResponse, path: str) -> bytes:
|
||||
data = bytearray()
|
||||
async for chunk in response.content.iter_chunked(DOWNLOAD_CHUNK_SIZE):
|
||||
async for chunk in response.content.iter_chunked(READ_CHUNK_SIZE):
|
||||
data.extend(chunk)
|
||||
if len(data) > MAX_CONFIG_BYTES:
|
||||
raise SyncError(f'{path} is larger than {MAX_CONFIG_BYTES} bytes')
|
||||
if len(data) > MAX_RESPONSE_BYTES:
|
||||
raise SyncError(f'{path} is larger than {MAX_RESPONSE_BYTES} bytes')
|
||||
return bytes(data)
|
||||
|
||||
|
||||
class PrimaryClient:
|
||||
"""Reads from a primary WROLPi: its sync API, and its media directory through Caddy."""
|
||||
"""Asks a primary WROLPi about itself (Test Connection) and lists its folders (to choose which are not copied).
|
||||
The sync itself is the Controller's, by rsync."""
|
||||
|
||||
def __init__(self, url: str):
|
||||
self.url = normalize_primary_url(url)
|
||||
|
|
@ -359,48 +232,24 @@ class PrimaryClient:
|
|||
|
||||
async def __aenter__(self):
|
||||
ssl_context = make_sync_ssl_context() if self.url.startswith('https') else False
|
||||
connector = aiohttp.TCPConnector(ssl=ssl_context, limit=DOWNLOAD_CONCURRENCY * 2)
|
||||
# A large file takes minutes; only a stalled read is an error.
|
||||
timeout = aiohttp.ClientTimeout(total=None, connect=30, sock_read=120)
|
||||
self.session = aiohttp.ClientSession(connector=connector, timeout=timeout, headers=DEFAULT_HTTP_HEADERS)
|
||||
timeout = aiohttp.ClientTimeout(total=60, connect=30)
|
||||
self.session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context), timeout=timeout,
|
||||
headers=DEFAULT_HTTP_HEADERS)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
await self.session.close()
|
||||
|
||||
def media_url(self, path: str) -> str:
|
||||
return f'{self.url}/media/{quote(path)}'
|
||||
|
||||
async def get_json(self, path: str, params: dict = None, headers: dict = None):
|
||||
async with self.session.get(f'{self.url}{path}', params=params, headers=headers) as response:
|
||||
async def get_info(self) -> dict:
|
||||
path = '/api/sync/info'
|
||||
async with self.session.get(f'{self.url}{path}') as response:
|
||||
if response.status != HTTPStatus.OK:
|
||||
raise SyncError(f'{path} returned HTTP {response.status}')
|
||||
return json.loads(await _read_limited(response, path))
|
||||
|
||||
async def get_info(self) -> dict:
|
||||
return await self.get_json('/api/sync/info')
|
||||
|
||||
async def iter_manifest(self, limit: int = MANIFEST_PAGE_LIMIT) -> AsyncGenerator[dict, None]:
|
||||
after_id = 0
|
||||
while True:
|
||||
page = await self.get_json('/api/sync/manifest', params=dict(after_id=after_id, limit=limit))
|
||||
yield page
|
||||
if page.get('next_after_id') is None:
|
||||
return
|
||||
after_id = page['next_after_id']
|
||||
|
||||
async def get_bytes(self, path: str) -> Optional[bytes]:
|
||||
"""A file in the primary's media directory, or None if it does not exist."""
|
||||
async with self.session.get(self.media_url(path)) as response:
|
||||
if response.status == HTTPStatus.NOT_FOUND:
|
||||
return None
|
||||
if response.status != HTTPStatus.OK:
|
||||
raise SyncError(f'{path} returned HTTP {response.status}')
|
||||
return await _read_limited(response, path)
|
||||
|
||||
async def list_directory(self, path: str) -> List[dict]:
|
||||
"""The entries of a directory in the primary's media directory (Caddy's browse listing), [] if missing."""
|
||||
url = self.media_url(path.rstrip('/') + '/')
|
||||
url = f'{self.url}/media/{quote(path.rstrip("/") + "/")}'
|
||||
async with self.session.get(url, headers={'Accept': 'application/json'}) as response:
|
||||
if response.status == HTTPStatus.NOT_FOUND:
|
||||
return []
|
||||
|
|
@ -408,218 +257,9 @@ class PrimaryClient:
|
|||
raise SyncError(f'{path} returned HTTP {response.status}')
|
||||
return json.loads(await _read_limited(response, path))
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def open_media(self, path: str, offset: int = 0):
|
||||
# Sizes and ranges count the file's bytes, so the file must not be compressed in transit.
|
||||
headers = {'Accept-Encoding': 'identity'}
|
||||
if offset:
|
||||
headers['Range'] = f'bytes={offset}-'
|
||||
async with self.session.get(self.media_url(path), headers=headers) as response:
|
||||
yield response
|
||||
|
||||
|
||||
async def download_file(client: PrimaryClient, file: ManifestFile, destination: pathlib.Path,
|
||||
on_bytes=None, before_replace: Callable[[pathlib.Path], None] = None) -> int:
|
||||
"""Download one file to `destination` with the primary's modification time. Returns the bytes received.
|
||||
|
||||
A partial file left by an earlier attempt is resumed. `before_replace` is called with the complete new version
|
||||
just before it takes the place of `destination`."""
|
||||
partial = partial_path(destination)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
offset = partial.stat().st_size if partial.is_file() else 0
|
||||
if offset > file.size:
|
||||
offset = 0
|
||||
remaining = file.size - offset
|
||||
if shutil.disk_usage(destination.parent).free - remaining < MIN_FREE_BYTES:
|
||||
raise SyncDiskFull(f'Not enough free space for {file.path}')
|
||||
|
||||
received = 0
|
||||
if remaining > 0 or not partial.is_file():
|
||||
async with client.open_media(file.path, offset) as response:
|
||||
if response.status == HTTPStatus.PARTIAL_CONTENT and offset:
|
||||
mode = 'ab'
|
||||
elif response.status == HTTPStatus.OK:
|
||||
mode, offset = 'wb', 0
|
||||
elif response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
|
||||
# The partial does not match the primary's file; start over next time.
|
||||
partial.unlink(missing_ok=True)
|
||||
raise SyncError(f'{file.path} changed on the primary')
|
||||
else:
|
||||
raise SyncError(f'{file.path} returned HTTP {response.status}')
|
||||
|
||||
if response.content_length is not None and response.content_length != file.size - offset:
|
||||
# The file changed on the primary after the manifest was read; the next sync gets it.
|
||||
raise SyncError(f'{file.path} changed on the primary')
|
||||
|
||||
with open(partial, mode) as fh:
|
||||
async for chunk in response.content.iter_chunked(DOWNLOAD_CHUNK_SIZE):
|
||||
if offset + received + len(chunk) > file.size:
|
||||
# Never write past the size the free-space check allowed for.
|
||||
fh.close()
|
||||
partial.unlink(missing_ok=True)
|
||||
raise SyncError(f'{file.path} is larger than the primary said')
|
||||
await asyncio.to_thread(fh.write, chunk)
|
||||
received += len(chunk)
|
||||
if on_bytes:
|
||||
on_bytes(len(chunk))
|
||||
|
||||
if (size := partial.stat().st_size) != file.size:
|
||||
raise SyncError(f'{file.path} is {size} bytes, the primary said {file.size}')
|
||||
os.utime(partial, (file.mtime, file.mtime))
|
||||
if before_replace:
|
||||
before_replace(partial)
|
||||
os.replace(partial, destination)
|
||||
return received
|
||||
|
||||
|
||||
def link_directories() -> List[tuple]:
|
||||
"""Where hard links of library files are kept, each as (directory, recursive): the Tags Directory, the
|
||||
playlists directory, and the directory of every playlist that has its own."""
|
||||
directories = [(get_tags_directory(), True), (get_playlists_directory(), True)]
|
||||
with get_db_session() as session:
|
||||
custom = session.execute(
|
||||
select(Collection.directory).where(Collection.kind == 'playlist', Collection.directory.isnot(None))
|
||||
).scalars().all()
|
||||
# A playlist's own directory may be any folder of the library; only the links directly in it are its own.
|
||||
directories.extend((pathlib.Path(i), False) for i in custom)
|
||||
return directories
|
||||
|
||||
|
||||
def _is_link_name(link_name: str, file_name: str) -> bool:
|
||||
"""True if `link_name` is what the Tags Directory (the file's name) or a playlist (the name after its position)
|
||||
calls a link to a file named `file_name`."""
|
||||
return link_name == file_name or bool(MANAGED_NAME.match(link_name)) and link_name[5:] == file_name
|
||||
|
||||
|
||||
def find_links(files: List[pathlib.Path]) -> Dict[pathlib.Path, List[pathlib.Path]]:
|
||||
"""The Tags Directory and playlist links of each of `files`: the files in `link_directories` that are the same
|
||||
inode as the file and are named for it. A file without other hard links has none and is left out."""
|
||||
inodes = dict()
|
||||
for file in files:
|
||||
try:
|
||||
stat = file.lstat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if stat.st_nlink > 1:
|
||||
# Library files may be hard links of each other, so several files can share an inode.
|
||||
inodes.setdefault((stat.st_dev, stat.st_ino), []).append(file)
|
||||
links = dict()
|
||||
if not inodes:
|
||||
return links
|
||||
|
||||
seen = set()
|
||||
for directory, recursive in link_directories():
|
||||
roots = (i[0] for i in os.walk(directory)) if recursive else (str(directory),)
|
||||
for root in roots:
|
||||
if root in seen:
|
||||
continue
|
||||
seen.add(root)
|
||||
try:
|
||||
names = os.listdir(root)
|
||||
except OSError:
|
||||
continue
|
||||
for name in names:
|
||||
path = pathlib.Path(root, name)
|
||||
try:
|
||||
stat = path.lstat()
|
||||
except OSError:
|
||||
continue
|
||||
for file in inodes.get((stat.st_dev, stat.st_ino), ()):
|
||||
if _is_link_name(name, file.name):
|
||||
links.setdefault(file, []).append(path)
|
||||
return links
|
||||
|
||||
|
||||
def move_links(links: List[pathlib.Path], old_version: pathlib.Path, new_version: pathlib.Path):
|
||||
"""Make each of `links` that is still a link to `old_version` a link to `new_version`.
|
||||
|
||||
The Tags Directory and a playlist refuse to delete a file that has no other link, because it may be a last
|
||||
copy. A link left on a replaced version is exactly that: once it is no longer wanted (the primary changed the
|
||||
file's tags, or reordered the playlist) it would stay forever."""
|
||||
try:
|
||||
old = old_version.lstat()
|
||||
except FileNotFoundError:
|
||||
return
|
||||
for link in links:
|
||||
# Short, so it fits wherever the link's own name does.
|
||||
temporary = link.with_name(f'.{uuid.uuid4().hex}.sync')
|
||||
try:
|
||||
stat = link.lstat()
|
||||
if (stat.st_dev, stat.st_ino) != (old.st_dev, old.st_ino):
|
||||
continue
|
||||
# Linked beside it, then renamed over it: the link is never missing.
|
||||
os.link(new_version, temporary)
|
||||
os.replace(temporary, link)
|
||||
except OSError as e:
|
||||
logger.error(f'Failed to move the link {repr(str(link))}: {e}')
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
# Secondary: configs.
|
||||
|
||||
|
||||
def _import_tags():
|
||||
from wrolpi.tags import import_tags_config
|
||||
import_tags_config()
|
||||
|
||||
|
||||
def _import_downloads():
|
||||
from wrolpi.downloader import get_download_manager_config
|
||||
get_download_manager_config().import_config()
|
||||
|
||||
|
||||
def _import_channels():
|
||||
from modules.videos.lib import import_channels_config
|
||||
import_channels_config()
|
||||
|
||||
|
||||
def _import_domains():
|
||||
from modules.archive.lib import import_domains_config
|
||||
import_domains_config()
|
||||
|
||||
|
||||
def _import_playlists():
|
||||
from wrolpi.collections.config import playlists_config
|
||||
from wrolpi.collections.sync import sync_playlists_directory
|
||||
playlists_config.import_config()
|
||||
sync_playlists_directory.activate_switch()
|
||||
|
||||
|
||||
def _import_zims():
|
||||
from modules.zim.lib import import_zims_config
|
||||
import_zims_config()
|
||||
|
||||
|
||||
def _import_plain(file_name: str):
|
||||
def import_():
|
||||
get_config_by_file_name(file_name).import_config()
|
||||
|
||||
return import_
|
||||
|
||||
|
||||
def _import_inventories():
|
||||
from modules.inventory.common import import_inventories_config
|
||||
import_inventories_config()
|
||||
|
||||
|
||||
# In the order of `import_all_db_configs`: Collections refer to Tags, and link to Downloads.
|
||||
SYNCED_CONFIG_IMPORTS = (
|
||||
('tags.yaml', _import_tags),
|
||||
('download_manager.yaml', _import_downloads),
|
||||
('channels.yaml', _import_channels),
|
||||
('domains.yaml', _import_domains),
|
||||
('playlists.yaml', _import_playlists),
|
||||
('zims.yaml', _import_zims),
|
||||
('map_pins.yaml', _import_plain('map_pins.yaml')),
|
||||
('maps.yaml', _import_plain('maps.yaml')),
|
||||
('bookmarks.yaml', _import_plain('bookmarks.yaml')),
|
||||
('videos_downloader.yaml', _import_plain('videos_downloader.yaml')),
|
||||
('archives_downloader.yaml', _import_plain('archives_downloader.yaml')),
|
||||
)
|
||||
INVENTORY_DIRECTORY = 'config/inventory'
|
||||
assert {i for i, _ in SYNCED_CONFIG_IMPORTS} | {'inventory/catalog.yaml'} == SYNCED_CONFIG_FILE_NAMES
|
||||
# Secondary: the primary's library settings.
|
||||
|
||||
|
||||
def _parse_config(data: bytes, name: str) -> dict:
|
||||
|
|
@ -629,66 +269,6 @@ def _parse_config(data: bytes, name: str) -> dict:
|
|||
return config
|
||||
|
||||
|
||||
def write_config_bytes(file: pathlib.Path, data: bytes, backup: pathlib.Path = None) -> bool:
|
||||
"""Replace `file` with the primary's bytes (keeping its version). Returns False if they are already the same.
|
||||
|
||||
The replaced file is copied to `backup` first (today's config backup), unless today's backup exists: like a save,
|
||||
the first copy of the day is the one kept, so a sync can be undone with a config backup import."""
|
||||
if file.is_file() and file.read_bytes() == data:
|
||||
return False
|
||||
if backup and file.is_file() and not backup.exists():
|
||||
backup.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(file, backup)
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = file.with_name(f'.{file.name}.sync')
|
||||
temporary.write_bytes(data)
|
||||
os.replace(temporary, file)
|
||||
return True
|
||||
|
||||
|
||||
async def sync_configs(client: PrimaryClient, primary_media_directory: str, reimport: bool) -> List[str]:
|
||||
"""Copy the primary's synced configs, then import those that changed (all of them when `reimport`, because newly
|
||||
downloaded files may let a config import entries it skipped before). Returns the imported configs."""
|
||||
imported = list()
|
||||
for file_name, import_ in SYNCED_CONFIG_IMPORTS:
|
||||
data = await client.get_bytes(f'config/{file_name}')
|
||||
if data is None:
|
||||
continue
|
||||
config = get_config_by_file_name(file_name)
|
||||
_parse_config(data, file_name)
|
||||
changed = write_config_bytes(config.get_file(), data, config._get_backup_filename())
|
||||
if changed or reimport:
|
||||
logger.info(f'Importing {file_name} from the primary')
|
||||
await asyncio.to_thread(import_)
|
||||
imported.append(file_name)
|
||||
|
||||
if await _sync_inventory_directory(client) or reimport:
|
||||
logger.info('Importing inventories from the primary')
|
||||
await asyncio.to_thread(_import_inventories)
|
||||
imported.append('inventory')
|
||||
|
||||
await _merge_wrolpi_config(client, primary_media_directory)
|
||||
return imported
|
||||
|
||||
|
||||
async def _sync_inventory_directory(client: PrimaryClient) -> bool:
|
||||
from modules.inventory.common import get_inventory_configs
|
||||
inventories = get_inventory_configs()
|
||||
directory = inventories.get_directory()
|
||||
changed = False
|
||||
for entry in await client.list_directory(INVENTORY_DIRECTORY):
|
||||
name = entry.get('name') or ''
|
||||
if entry.get('is_dir') or not name.endswith('.yaml') or '/' in name or name.startswith('.'):
|
||||
continue
|
||||
data = await client.get_bytes(f'{INVENTORY_DIRECTORY}/{name}')
|
||||
if data is None:
|
||||
continue
|
||||
_parse_config(data, name)
|
||||
backup = inventories._get_backup_filename(name.removesuffix('.yaml'))
|
||||
changed = write_config_bytes(directory / name, data, backup) or changed
|
||||
return changed
|
||||
|
||||
|
||||
def merge_wrolpi_config(primary_config: dict, primary_media_directory: str) -> dict:
|
||||
"""The library settings of the primary's wrolpi.yaml that differ from this WROLPi's."""
|
||||
wrolpi_config = get_wrolpi_config()
|
||||
|
|
@ -743,226 +323,8 @@ def _relative_directories(directories: List[str], primary_media_directory: str)
|
|||
return relative
|
||||
|
||||
|
||||
async def _merge_wrolpi_config(client: PrimaryClient, primary_media_directory: str):
|
||||
data = await client.get_bytes('config/wrolpi.yaml')
|
||||
if data is None:
|
||||
return
|
||||
if changes := merge_wrolpi_config(_parse_config(data, 'wrolpi.yaml'), primary_media_directory):
|
||||
logger.info(f'Applying library settings from the primary: {sorted(changes)}')
|
||||
get_wrolpi_config().update(changes)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
# Secondary: the sync.
|
||||
|
||||
|
||||
def sync_skip_reason() -> Optional[str]:
|
||||
"""Why a sync cannot start now, or None."""
|
||||
config = get_sync_config()
|
||||
if not is_secondary():
|
||||
return 'This WROLPi is not a secondary'
|
||||
if wrol_mode_enabled():
|
||||
return 'WROL Mode is enabled'
|
||||
if not config.primary_url:
|
||||
return 'No primary is configured'
|
||||
if not PYTEST:
|
||||
if not flags.db_up.is_set():
|
||||
return 'The database is not up'
|
||||
if not flags.refresh_complete.is_set():
|
||||
return 'The first refresh has not finished'
|
||||
if get_wrolpi_config().require_media_mounted and not flags.media_mounted.is_set():
|
||||
return 'The media directory is not mounted'
|
||||
if flags.sync_running.is_set():
|
||||
return 'A sync is already running'
|
||||
return None
|
||||
|
||||
|
||||
def request_sync(force: bool = False):
|
||||
update_sync_status(requested=True, force=force)
|
||||
|
||||
|
||||
def request_sync_cancel():
|
||||
"""Stop a running sync between pages, and forget a requested one."""
|
||||
from wrolpi.api_utils import api_app
|
||||
if 'sync' not in api_app.shared_ctx.status:
|
||||
return
|
||||
status = get_sync_status()
|
||||
if status['state'] == 'running' or status['requested']:
|
||||
update_sync_status(cancel_requested=status['state'] == 'running', requested=False)
|
||||
|
||||
|
||||
def _check_cancel():
|
||||
if get_sync_status()['cancel_requested']:
|
||||
raise SyncCancelled('The sync was cancelled')
|
||||
if wrol_mode_enabled():
|
||||
raise SyncCancelled('WROL Mode was enabled')
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def lock_api():
|
||||
"""Refuse API requests (see `check_sync_api_lock`) while the library is replaced."""
|
||||
from wrolpi.api_utils import api_app
|
||||
with flags.sync_running:
|
||||
api_app.shared_ctx.sync_api_locked.set()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
api_app.shared_ctx.sync_api_locked.clear()
|
||||
|
||||
|
||||
async def run_sync(force: bool = False) -> dict:
|
||||
"""Copy the primary's library to this secondary. Returns the final status."""
|
||||
if reason := sync_skip_reason():
|
||||
logger.info(f'Not syncing: {reason}')
|
||||
return update_sync_status(last_skip_reason=reason, force=False)
|
||||
|
||||
url = get_sync_config().primary_url
|
||||
started_at = now()
|
||||
update_sync_status(
|
||||
state='running', phase='files', force=False, cancel_requested=False, started_at=started_at.isoformat(),
|
||||
finished_at=None, last_error=None, last_skip_reason=None, primary_url=url, primary_version=None,
|
||||
groups_seen=0, max_id=0, files_checked=0, files_to_download=0, files_downloaded=0, bytes_downloaded=0,
|
||||
files_failed=0, files_excluded=0, configs_imported=[], disk_full=False,
|
||||
)
|
||||
logger.warning(f'Syncing from the primary {url}')
|
||||
try:
|
||||
with lock_api():
|
||||
async with PrimaryClient(url) as client:
|
||||
info = await client.get_info()
|
||||
if info.get('role') != 'primary':
|
||||
raise SyncError(f'{url} is not a primary WROLPi')
|
||||
if (info.get('version') or '').strip() != __version__.strip():
|
||||
logger.warning(f'The primary runs WROLPi {info.get("version")}, this is {__version__.strip()}')
|
||||
update_sync_status(primary_version=(info.get('version') or '').strip() or None)
|
||||
|
||||
downloaded = await _sync_files(client)
|
||||
_check_cancel()
|
||||
|
||||
update_sync_status(phase='configs')
|
||||
primary_media_directory = info.get('media_directory') or ''
|
||||
imported = await sync_configs(client, primary_media_directory, reimport=bool(downloaded) or force)
|
||||
except SyncCancelled as e:
|
||||
logger.warning(f'Sync stopped: {e}')
|
||||
return update_sync_status(state='idle', phase=None, finished_at=now().isoformat(), last_error=str(e),
|
||||
cancel_requested=False)
|
||||
except Exception as e:
|
||||
logger.error('Sync failed', exc_info=e)
|
||||
Events.send_sync_failed(f'Sync from the primary failed: {e}')
|
||||
return update_sync_status(state='error', phase=None, finished_at=now().isoformat(), last_error=str(e),
|
||||
cancel_requested=False)
|
||||
|
||||
status = get_sync_status()
|
||||
finished_at = now()
|
||||
failed = status['files_failed']
|
||||
message = f'Synced {len(downloaded)} files from the primary'
|
||||
if failed:
|
||||
message += f'; {failed} failed'
|
||||
if status['disk_full']:
|
||||
message += '; the drive is full'
|
||||
logger.warning(f'{message} in {(finished_at - started_at).total_seconds():.0f}s')
|
||||
if downloaded or failed or imported:
|
||||
Events.send_sync_completed(message)
|
||||
last_success_at = finished_at.isoformat() if not failed and not status['disk_full'] else status['last_success_at']
|
||||
return update_sync_status(state='idle', phase=None, finished_at=finished_at.isoformat(),
|
||||
last_success_at=last_success_at, configs_imported=imported,
|
||||
last_error=f'{failed} files failed' if failed else None)
|
||||
|
||||
|
||||
async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]:
|
||||
"""Download the files this secondary lacks, and refresh them a page at a time. Returns the files downloaded."""
|
||||
media_directory = get_media_directory()
|
||||
downloaded = list()
|
||||
groups_seen = files_checked = files_to_download = files_failed = files_excluded = 0
|
||||
excluded_directories = set(get_sync_config().sync_excluded_directories)
|
||||
bytes_downloaded = 0
|
||||
disk_full = False
|
||||
|
||||
last_report = 0.0
|
||||
|
||||
def report(force: bool = False):
|
||||
nonlocal last_report
|
||||
if force or time.monotonic() - last_report >= PROGRESS_INTERVAL_SECONDS:
|
||||
last_report = time.monotonic()
|
||||
update_sync_status(groups_seen=groups_seen, files_checked=files_checked,
|
||||
files_to_download=files_to_download, files_downloaded=len(downloaded),
|
||||
bytes_downloaded=bytes_downloaded, files_failed=files_failed,
|
||||
files_excluded=files_excluded, disk_full=disk_full)
|
||||
|
||||
def count_bytes(n: int):
|
||||
nonlocal bytes_downloaded
|
||||
bytes_downloaded += n
|
||||
report()
|
||||
|
||||
async for page in client.iter_manifest():
|
||||
_check_cancel()
|
||||
wanted = list()
|
||||
for group in page['groups']:
|
||||
groups_seen += 1
|
||||
for entry in group['paths']:
|
||||
files_checked += 1
|
||||
file = ManifestFile(entry['path'], int(entry['size']), float(entry['mtime']))
|
||||
if excluded_directories and is_excluded(file.path, excluded_directories):
|
||||
files_excluded += 1
|
||||
continue
|
||||
destination = resolve_destination(media_directory, file.path)
|
||||
if destination is None:
|
||||
logger.warning(f'Refusing to sync {repr(file.path)}')
|
||||
files_failed += 1
|
||||
continue
|
||||
if needs_download(destination, file):
|
||||
wanted.append((file, destination))
|
||||
files_to_download += len(wanted)
|
||||
update_sync_status(max_id=page.get('max_id') or 0)
|
||||
report(force=True)
|
||||
|
||||
page_downloaded = list()
|
||||
if wanted and not disk_full:
|
||||
semaphore = asyncio.Semaphore(DOWNLOAD_CONCURRENCY)
|
||||
# A file's links move to its new version before the new version takes its place, so a sync that is
|
||||
# interrupted in between downloads nothing again and still leaves no link on the old version.
|
||||
links = await asyncio.to_thread(find_links, [i for _, i in wanted])
|
||||
|
||||
async def download(file_: ManifestFile, destination_: pathlib.Path):
|
||||
nonlocal disk_full
|
||||
async with semaphore:
|
||||
if disk_full:
|
||||
return None
|
||||
try:
|
||||
before_replace = None
|
||||
if destination_ in links:
|
||||
def before_replace(new_version: pathlib.Path):
|
||||
move_links(links[destination_], destination_, new_version)
|
||||
await download_file(client, file_, destination_, count_bytes, before_replace)
|
||||
downloaded.append(destination_)
|
||||
return destination_
|
||||
except SyncDiskFull as e:
|
||||
disk_full = True
|
||||
logger.error(str(e))
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to sync {repr(file_.path)}: {e}')
|
||||
return False
|
||||
|
||||
results = await asyncio.gather(*(download(f, d) for f, d in wanted))
|
||||
page_downloaded = [i for i in results if i]
|
||||
files_failed += sum(1 for i in results if i is False)
|
||||
|
||||
update_sync_status(max_id=page.get('max_id') or 0)
|
||||
report(force=True)
|
||||
|
||||
if page_downloaded:
|
||||
await _refresh(page_downloaded)
|
||||
|
||||
return downloaded
|
||||
|
||||
|
||||
async def _refresh(paths: List[pathlib.Path]):
|
||||
from wrolpi.files.worker import file_worker, FileWorkerJobFailed
|
||||
job_id = file_worker.queue_refresh(paths, send_events=False)
|
||||
try:
|
||||
await file_worker.wait_for_job(job_id, timeout=REFRESH_TIMEOUT_SECONDS)
|
||||
except (FileWorkerJobFailed, TimeoutError) as e:
|
||||
# The files are in place; a later refresh indexes them.
|
||||
logger.error(f'Failed to refresh synced files: {e}')
|
||||
# The daily window (the backup drive's; the Controller has its own for the sync).
|
||||
|
||||
|
||||
def window_is_due(status: dict, local_now: datetime, start: Optional[str], end: Optional[str],
|
||||
|
|
@ -979,24 +341,3 @@ def window_is_due(status: dict, local_now: datetime, start: Optional[str], end:
|
|||
return (local_now - finished_at).total_seconds() >= repeat_seconds
|
||||
|
||||
|
||||
def sync_is_due(status: dict, local_now: datetime) -> bool:
|
||||
"""True if the sync window is open and the last sync finished long enough ago."""
|
||||
config = get_sync_config()
|
||||
return window_is_due(status, local_now, config.window_start, config.window_end, SYNC_WINDOW_REPEAT_SECONDS)
|
||||
|
||||
|
||||
async def sync_tick():
|
||||
"""Start a sync when one was requested, or when it is due."""
|
||||
if not is_secondary():
|
||||
return
|
||||
status = get_sync_status()
|
||||
if wrol_mode_enabled():
|
||||
if status['requested']:
|
||||
update_sync_status(requested=False, last_skip_reason='WROL Mode is enabled')
|
||||
return
|
||||
from wrolpi.downloader import DownloadManager
|
||||
if not status['requested'] and not sync_is_due(status, DownloadManager._get_local_now()):
|
||||
return
|
||||
force = status['force']
|
||||
update_sync_status(requested=False)
|
||||
await run_sync(force=force)
|
||||
|
|
|
|||
73
wrolpi/sync/notify.py
Normal file
73
wrolpi/sync/notify.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
"""Notifications for a secondary's syncs.
|
||||
|
||||
The Controller runs a sync (controller/lib/sync.py) and stops this API while it copies, so it cannot send the API's
|
||||
events itself. It records each sync's outcome in `config/sync_state.yaml`; the API reads that record and sends one
|
||||
event per finished sync. What it has announced is kept in its own file, which a sync never touches, so a restart
|
||||
(and every sync restarts the API) neither repeats nor loses an announcement.
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
import yaml
|
||||
|
||||
from wrolpi.common import logger, get_media_directory, is_secondary
|
||||
from wrolpi.events import Events
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
STATE_FILE = 'config/sync_state.yaml'
|
||||
# Under config/.sync, which a sync never copies over or deletes.
|
||||
ANNOUNCED_FILE = 'config/.sync/announced.yaml'
|
||||
# A sync that was stopped on purpose is not news.
|
||||
STOPPED = ('The sync was cancelled', 'WROL Mode was enabled')
|
||||
|
||||
|
||||
def _read(path: str) -> dict:
|
||||
try:
|
||||
data = yaml.safe_load((get_media_directory() / path).read_text())
|
||||
except (OSError, yaml.YAMLError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def sync_message(state: dict) -> Optional[tuple]:
|
||||
"""The event a finished sync deserves, as (kind, message), or None."""
|
||||
error = state.get('last_error')
|
||||
if state.get('state') == 'error':
|
||||
return 'failed', f'Sync from the primary failed: {error}'
|
||||
if error in STOPPED:
|
||||
return None
|
||||
copied, deleted = int(state.get('files_transferred') or 0), int(state.get('files_deleted') or 0)
|
||||
warnings = state.get('warnings') or []
|
||||
if not copied and not deleted and not warnings:
|
||||
# Within the window a sync runs every 15 minutes; one that changed nothing is not news.
|
||||
return None
|
||||
message = f'Synced {copied} files from the primary'
|
||||
if deleted:
|
||||
message += f'; {deleted} moved to the trash'
|
||||
if warnings:
|
||||
message += f'; {warnings[0]}'
|
||||
return 'completed', message
|
||||
|
||||
|
||||
def notify_tick():
|
||||
"""Send the event of a sync that finished since the last one announced."""
|
||||
if not is_secondary():
|
||||
return
|
||||
state = _read(STATE_FILE)
|
||||
finished_at = state.get('finished_at')
|
||||
if not finished_at or state.get('state') == 'running':
|
||||
return
|
||||
if _read(ANNOUNCED_FILE).get('finished_at') == finished_at:
|
||||
return
|
||||
if found := sync_message(state):
|
||||
kind, message = found
|
||||
if kind == 'failed':
|
||||
Events.send_sync_failed(message)
|
||||
else:
|
||||
Events.send_sync_completed(message)
|
||||
announced = get_media_directory() / ANNOUNCED_FILE
|
||||
try:
|
||||
announced.parent.mkdir(parents=True, exist_ok=True)
|
||||
announced.write_text(yaml.safe_dump(dict(finished_at=finished_at)))
|
||||
except OSError as e:
|
||||
logger.warning(f'Could not record the announced sync: {e}')
|
||||
|
|
@ -4,20 +4,6 @@ from typing import List, Optional
|
|||
from wrolpi.errors import ValidationError
|
||||
|
||||
|
||||
@dataclass
|
||||
class ManifestRequest:
|
||||
after_id: int = 0
|
||||
limit: int = 500
|
||||
|
||||
def __post_init__(self):
|
||||
self.after_id = int(self.after_id)
|
||||
self.limit = int(self.limit)
|
||||
if self.after_id < 0:
|
||||
raise ValidationError('after_id must not be negative')
|
||||
if not 1 <= self.limit <= 5000:
|
||||
raise ValidationError('limit must be between 1 and 5000')
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncDirectoriesRequest:
|
||||
# The folder whose sub-folders are listed, relative to the media directory ('' is the media directory).
|
||||
|
|
|
|||
|
|
@ -1,53 +1,11 @@
|
|||
"""The shared progress records of a secondary WROLPi's sync, `api_app.shared_ctx.status['sync']`, and of a backup to
|
||||
a second drive, `api_app.shared_ctx.status['backup']`.
|
||||
"""The shared progress records of a backup to a second drive, `api_app.shared_ctx.status['backup']`, and of a
|
||||
primary's database snapshot for its secondaries. (A secondary's sync is the Controller's: controller/lib/sync.py.)
|
||||
|
||||
They live in a manager dict, so every read returns a copy: change them only with `update_sync_status` and
|
||||
`update_backup_status`.
|
||||
They live in a manager dict, so every read returns a copy: change them only with `update_backup_status` and
|
||||
`update_snapshot_status`.
|
||||
"""
|
||||
|
||||
|
||||
def default_sync_status() -> dict:
|
||||
return dict(
|
||||
state='idle',
|
||||
phase=None,
|
||||
requested=False,
|
||||
force=False,
|
||||
cancel_requested=False,
|
||||
started_at=None,
|
||||
finished_at=None,
|
||||
last_success_at=None,
|
||||
last_error=None,
|
||||
last_skip_reason=None,
|
||||
primary_url=None,
|
||||
primary_version=None,
|
||||
groups_seen=0,
|
||||
max_id=0,
|
||||
files_checked=0,
|
||||
files_to_download=0,
|
||||
files_downloaded=0,
|
||||
bytes_downloaded=0,
|
||||
files_failed=0,
|
||||
files_excluded=0,
|
||||
configs_imported=[],
|
||||
disk_full=False,
|
||||
)
|
||||
|
||||
|
||||
def get_sync_status() -> dict:
|
||||
from wrolpi.api_utils import api_app
|
||||
return dict(api_app.shared_ctx.status.get('sync') or default_sync_status())
|
||||
|
||||
|
||||
def update_sync_status(**changes) -> dict:
|
||||
"""Merge `changes` into the shared sync status. Only the perpetual process and the sync endpoints write it, and
|
||||
the endpoints only set the request/cancel keys, so a lost update can only delay a request by one tick."""
|
||||
from wrolpi.api_utils import api_app
|
||||
status = get_sync_status()
|
||||
status.update(changes)
|
||||
api_app.shared_ctx.status['sync'] = status
|
||||
return status
|
||||
|
||||
|
||||
def default_snapshot_status() -> dict:
|
||||
return dict(
|
||||
state='idle',
|
||||
|
|
@ -73,7 +31,8 @@ def get_snapshot_status() -> dict:
|
|||
|
||||
|
||||
def update_snapshot_status(**changes) -> dict:
|
||||
"""Merge `changes` into the shared snapshot status (see `update_sync_status`)."""
|
||||
"""Merge `changes` into the shared snapshot status. Only the perpetual process and the snapshot endpoint write
|
||||
it, and the endpoint only sets `requested`, so a lost update can only delay a request by one tick."""
|
||||
from wrolpi.api_utils import api_app
|
||||
status = get_snapshot_status()
|
||||
status.update(changes)
|
||||
|
|
@ -106,7 +65,7 @@ def get_backup_status() -> dict:
|
|||
|
||||
|
||||
def update_backup_status(**changes) -> dict:
|
||||
"""Merge `changes` into the shared backup status (see `update_sync_status`)."""
|
||||
"""Merge `changes` into the shared backup status (see `update_snapshot_status`)."""
|
||||
from wrolpi.api_utils import api_app
|
||||
status = get_backup_status()
|
||||
status.update(changes)
|
||||
|
|
|
|||
|
|
@ -5,38 +5,22 @@ from http import HTTPStatus
|
|||
import pytest
|
||||
import yaml
|
||||
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.common import get_wrolpi_config, enable_wrol_mode
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.sync import api as sync_api, backup, lib
|
||||
from wrolpi.sync import api as sync_api, backup, notify, snapshot
|
||||
from wrolpi.sync.config import get_sync_config
|
||||
from wrolpi.sync.status import get_sync_status, update_sync_status
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_info_and_manifest(async_client, test_session, test_wrolpi_config, make_files_structure,
|
||||
refresh_files):
|
||||
make_files_structure({'docs/a.txt': 'a', 'docs/b.txt': 'b'})
|
||||
await refresh_files()
|
||||
|
||||
async def test_info(async_client, test_session, test_wrolpi_config):
|
||||
request, response = await async_client.get('/api/sync/info')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert set(response.json) == {'version', 'role', 'media_directory', 'database_revision', 'media_mounted'}
|
||||
assert response.json['role'] == 'primary'
|
||||
assert response.json['max_file_group_id'] > 0
|
||||
|
||||
request, response = await async_client.get('/api/sync/manifest?after_id=0&limit=1')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['groups']) == 1
|
||||
after_id = response.json['next_after_id']
|
||||
assert after_id
|
||||
|
||||
request, response = await async_client.get(f'/api/sync/manifest?after_id={after_id}&limit=1')
|
||||
paths = [p['path'] for g in response.json['groups'] for p in g['paths']]
|
||||
assert len(paths) == 1 and paths[0].startswith('docs/')
|
||||
|
||||
request, response = await async_client.get('/api/sync/manifest?limit=0')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
# The old engine's manifest is gone; a secondary copies with rsync.
|
||||
request, response = await async_client.get('/api/sync/manifest')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class FakeControllerResponse:
|
||||
|
|
@ -91,19 +75,19 @@ async def test_run(async_client, test_wrolpi_config, make_secondary, monkeypatch
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_engine_does_not_run(async_client, test_wrolpi_config, make_secondary, monkeypatch):
|
||||
"""Only the Controller syncs now; the API's loop runs the backup drive and nothing else."""
|
||||
make_secondary('https://primary.example')
|
||||
lib.request_sync()
|
||||
async def test_worker_runs_the_backup_and_the_snapshot(async_client, test_wrolpi_config, monkeypatch):
|
||||
"""The API's loop makes the backup and the primary's database snapshot, and announces the syncs the Controller
|
||||
finished; a secondary's sync itself is the Controller's."""
|
||||
called = []
|
||||
|
||||
async def tick(name):
|
||||
called.append(name)
|
||||
|
||||
monkeypatch.setattr(lib, 'sync_tick', lambda: tick('sync'))
|
||||
monkeypatch.setattr(backup, 'backup_tick', lambda: tick('backup'))
|
||||
monkeypatch.setattr(snapshot, 'snapshot_tick', lambda: tick('snapshot'))
|
||||
monkeypatch.setattr(notify, 'notify_tick', lambda: called.append('notify'))
|
||||
await sync_api.perpetual_sync_worker()
|
||||
assert called == ['backup']
|
||||
assert called == ['backup', 'snapshot', 'notify']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -215,56 +199,14 @@ async def test_directories_primary_unreachable(async_client, test_wrolpi_config,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_locked_while_syncing(async_client, test_wrolpi_config, flags_lock, make_secondary, monkeypatch):
|
||||
"""While a secondary syncs, its API refuses everything but the sync's own progress."""
|
||||
async def test_wrol_mode_refuses_a_sync(async_client, test_wrolpi_config, test_download_manager, make_secondary,
|
||||
monkeypatch):
|
||||
"""WROL Mode means the secondary is in use: no sync is asked of the Controller. (The Controller itself also
|
||||
refuses, and stops a sync at its next step, when it finds WROL Mode enabled.)"""
|
||||
make_secondary()
|
||||
with lib.lock_api():
|
||||
request, response = await async_client.get('/api/files/search?search_str=foo')
|
||||
assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
|
||||
assert response.json['code'] == 'SYNCING'
|
||||
request, response = await async_client.post('/api/tag', content=json.dumps(dict(name='one')))
|
||||
assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
|
||||
|
||||
for path in ('/api/status', '/api/sync/status', '/api/settings', '/api/echo'):
|
||||
request, response = await async_client.get(path)
|
||||
assert response.status_code == HTTPStatus.OK, path
|
||||
fake_controller(monkeypatch, status=HTTPStatus.ACCEPTED)
|
||||
request, response = await async_client.post('/api/sync/cancel')
|
||||
assert response.status_code == HTTPStatus.ACCEPTED
|
||||
# Nothing may change the sync while it runs, except stopping it.
|
||||
request, response = await async_client.put('/api/sync/settings', content=json.dumps(dict(role='primary')))
|
||||
assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
|
||||
request, response = await async_client.post('/api/sync/run', content='{}')
|
||||
assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
|
||||
|
||||
request, response = await async_client.get('/api/status')
|
||||
assert response.json['flags']['sync_running'] is True
|
||||
|
||||
request, response = await async_client.get('/api/settings')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert not api_app.shared_ctx.sync_api_locked.is_set()
|
||||
assert not flags.sync_running.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_lock_event_does_not_lock(async_client, test_wrolpi_config, flags_lock):
|
||||
"""The Event outlives a perpetual process that died mid-sync; the cleared Flag unlocks the API."""
|
||||
api_app.shared_ctx.sync_api_locked.set()
|
||||
try:
|
||||
request, response = await async_client.get('/api/settings')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
finally:
|
||||
api_app.shared_ctx.sync_api_locked.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrol_mode_cancels_sync(async_client, test_wrolpi_config, test_download_manager, make_secondary):
|
||||
"""WROL Mode means the secondary is in use: a running sync stops, and none start."""
|
||||
make_secondary()
|
||||
update_sync_status(state='running')
|
||||
calls = fake_controller(monkeypatch, status=HTTPStatus.ACCEPTED)
|
||||
enable_wrol_mode()
|
||||
assert get_sync_status()['cancel_requested'] is True
|
||||
assert lib.sync_skip_reason() == 'WROL Mode is enabled'
|
||||
|
||||
request, response = await async_client.post('/api/sync/run', content='{}')
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN
|
||||
assert calls == []
|
||||
|
|
|
|||
|
|
@ -403,18 +403,15 @@ def test_backup_is_due_hourly_in_window(test_wrolpi_config):
|
|||
|
||||
|
||||
def test_backup_and_sync_windows_are_separate(test_wrolpi_config):
|
||||
"""The sync window (a secondary's sync) does not start backups, and the backup window does not start syncs."""
|
||||
from wrolpi.sync.lib import sync_is_due
|
||||
"""The sync window (a secondary's sync, which the Controller schedules) does not start backups."""
|
||||
local_now = datetime(2026, 11, 5, 3, 0, tzinfo=timezone.utc)
|
||||
never = dict(finished_at=None)
|
||||
|
||||
get_sync_config()._config.update(window_start='01:00', window_end='05:00')
|
||||
assert sync_is_due(never, local_now)
|
||||
assert not backup.backup_is_due(never, local_now)
|
||||
|
||||
get_sync_config()._config.update(window_start=None, window_end=None,
|
||||
backup_window_start='02:00', backup_window_end='04:00')
|
||||
assert not sync_is_due(never, local_now)
|
||||
assert backup.backup_is_due(never, local_now)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,698 +0,0 @@
|
|||
import os
|
||||
import pathlib
|
||||
from collections import namedtuple
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.collections.sync import get_playlists_directory, _safe_unlink_managed
|
||||
from wrolpi.common import get_wrolpi_config, set_test_media_directory
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import InvalidConfig
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.sync import lib
|
||||
from wrolpi.sync.config import get_sync_config
|
||||
from wrolpi.sync.conftest import manifest_page
|
||||
from wrolpi.sync.status import get_sync_status, update_sync_status
|
||||
from wrolpi.tags import get_tags_config, Tag, TagFile, _delete_extra_tags_directory_paths
|
||||
|
||||
|
||||
@pytest.mark.parametrize('path, allowed', [
|
||||
('videos/channel/video.mp4', True),
|
||||
('a.txt', True),
|
||||
('../escape.txt', False),
|
||||
('videos/../../escape.txt', False),
|
||||
('/etc/passwd', False),
|
||||
('videos/.hidden/video.mp4', False),
|
||||
('.hidden.txt', False),
|
||||
('config/wrolpi.db', False),
|
||||
('config/tags.yaml', False),
|
||||
('tags/one/video.mp4', False),
|
||||
# A drive that ignores case (exFAT, NTFS, macOS) would write these into the real config and tags folders.
|
||||
('Config/wrolpi.db', False),
|
||||
('CONFIG/tags.yaml', False),
|
||||
('Tags/one/video.mp4', False),
|
||||
('videos\\video.mp4', False),
|
||||
('', False),
|
||||
])
|
||||
def test_resolve_destination(test_directory, path, allowed):
|
||||
destination = lib.resolve_destination(test_directory, path)
|
||||
if allowed:
|
||||
assert destination == test_directory / path
|
||||
else:
|
||||
assert destination is None
|
||||
|
||||
|
||||
def test_resolve_destination_refuses_symlinks(test_directory):
|
||||
"""A sync never writes through a symlink; it could point outside the media directory."""
|
||||
outside = test_directory / 'outside'
|
||||
outside.mkdir()
|
||||
(test_directory / 'videos').symlink_to(outside)
|
||||
assert lib.resolve_destination(test_directory, 'videos/video.mp4') is None
|
||||
|
||||
(test_directory / 'docs').mkdir()
|
||||
(test_directory / 'docs/link.txt').symlink_to(outside / 'target.txt')
|
||||
assert lib.resolve_destination(test_directory, 'docs/link.txt') is None
|
||||
|
||||
|
||||
def test_needs_download(test_directory):
|
||||
path = test_directory / 'a.txt'
|
||||
file = lib.ManifestFile('a.txt', 5, 1_700_000_000.0)
|
||||
assert lib.needs_download(path, file), 'a missing file is downloaded'
|
||||
|
||||
path.write_text('12345')
|
||||
os.utime(path, (file.mtime, file.mtime))
|
||||
assert not lib.needs_download(path, file)
|
||||
|
||||
os.utime(path, (file.mtime + 1.5, file.mtime + 1.5))
|
||||
assert not lib.needs_download(path, file), 'within the 2 second tolerance of exFAT'
|
||||
|
||||
os.utime(path, (file.mtime + 3, file.mtime + 3))
|
||||
assert lib.needs_download(path, file), 'a different modification time is downloaded'
|
||||
|
||||
os.utime(path, (file.mtime, file.mtime))
|
||||
path.write_text('1234')
|
||||
os.utime(path, (file.mtime, file.mtime))
|
||||
assert lib.needs_download(path, file), 'a different size is downloaded'
|
||||
|
||||
|
||||
Primary = namedtuple('Primary', 'directory url state')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def primary(test_directory, secondary_directory, primary_media_server) -> Primary:
|
||||
"""A primary whose media is `test_directory`, served like Caddy."""
|
||||
url, state = primary_media_server(test_directory)
|
||||
return Primary(test_directory, url, state)
|
||||
|
||||
|
||||
def write_primary_file(primary: Primary, path: str, content: bytes, mtime: float = 1_700_000_000.0) \
|
||||
-> lib.ManifestFile:
|
||||
file = primary.directory / path
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file.write_bytes(content)
|
||||
os.utime(file, (mtime, mtime))
|
||||
return lib.ManifestFile(path, len(content), mtime)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_file(primary, secondary_directory):
|
||||
"""A downloaded file has the primary's bytes and modification time, and no partial is left."""
|
||||
file = write_primary_file(primary, 'videos/channel/video name.mp4', b'video bytes' * 1000)
|
||||
destination = secondary_directory / file.path
|
||||
|
||||
async with lib.PrimaryClient(primary.url) as client:
|
||||
received = await lib.download_file(client, file, destination)
|
||||
|
||||
assert received == file.size
|
||||
assert destination.read_bytes() == (primary.directory / file.path).read_bytes()
|
||||
assert destination.stat().st_mtime == file.mtime
|
||||
assert not lib.partial_path(destination).exists()
|
||||
assert not lib.needs_download(destination, file)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_file_resumes(primary, secondary_directory):
|
||||
"""A partial left by an interrupted download is resumed with a Range request."""
|
||||
content = bytes(range(256)) * 400
|
||||
file = write_primary_file(primary, 'big.bin', content)
|
||||
destination = secondary_directory / file.path
|
||||
lib.partial_path(destination).write_bytes(content[:30_000])
|
||||
|
||||
async with lib.PrimaryClient(primary.url) as client:
|
||||
received = await lib.download_file(client, file, destination)
|
||||
|
||||
assert received == len(content) - 30_000
|
||||
assert ('/media/big.bin', 'bytes=30000-') in primary.state['requests']
|
||||
assert destination.read_bytes() == content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_file_restarts_without_range(primary, secondary_directory):
|
||||
"""A server that ignores Range sends the whole file; the partial is replaced, not appended to."""
|
||||
primary.state['honor_range'] = False
|
||||
content = b'abcdefghij' * 100
|
||||
file = write_primary_file(primary, 'a.bin', content)
|
||||
destination = secondary_directory / file.path
|
||||
lib.partial_path(destination).write_bytes(content[:500])
|
||||
|
||||
async with lib.PrimaryClient(primary.url) as client:
|
||||
await lib.download_file(client, file, destination)
|
||||
|
||||
assert destination.read_bytes() == content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_file_changed_on_primary(primary, secondary_directory):
|
||||
"""A file whose size changed after the manifest was read is not written; the next sync gets it."""
|
||||
file = write_primary_file(primary, 'a.txt', b'old')
|
||||
(primary.directory / 'a.txt').write_bytes(b'new and longer')
|
||||
destination = secondary_directory / file.path
|
||||
|
||||
async with lib.PrimaryClient(primary.url) as client:
|
||||
with pytest.raises(lib.SyncError):
|
||||
await lib.download_file(client, file, destination)
|
||||
|
||||
assert not destination.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_file_larger_than_manifest(primary, secondary_directory):
|
||||
"""A primary that sends more bytes than its manifest said (no Content-Length to catch it) is not written past
|
||||
the manifest's size, which is what the free-space check allowed for."""
|
||||
primary.state['omit_length'] = True
|
||||
write_primary_file(primary, 'a.bin', b'x' * 5 * lib.DOWNLOAD_CHUNK_SIZE)
|
||||
file = lib.ManifestFile('a.bin', 3000, 1_700_000_000.0)
|
||||
destination = secondary_directory / file.path
|
||||
|
||||
async with lib.PrimaryClient(primary.url) as client:
|
||||
with pytest.raises(lib.SyncError, match='larger than the primary said'):
|
||||
await lib.download_file(client, file, destination)
|
||||
|
||||
assert not destination.exists()
|
||||
assert not lib.partial_path(destination).exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_file_disk_full(primary, secondary_directory, monkeypatch):
|
||||
"""A file that would leave less than the minimum free space is not downloaded; its partial is kept."""
|
||||
file = write_primary_file(primary, 'a.bin', b'x' * 1000)
|
||||
destination = secondary_directory / file.path
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
lib.partial_path(destination).write_bytes(b'x' * 10)
|
||||
|
||||
usage = namedtuple('usage', 'total used free')
|
||||
monkeypatch.setattr(lib.shutil, 'disk_usage', lambda _: usage(10, 10, lib.MIN_FREE_BYTES))
|
||||
|
||||
async with lib.PrimaryClient(primary.url) as client:
|
||||
with pytest.raises(lib.SyncDiskFull):
|
||||
await lib.download_file(client, file, destination)
|
||||
|
||||
assert lib.partial_path(destination).read_bytes() == b'x' * 10
|
||||
assert primary.state['requests'] == []
|
||||
|
||||
|
||||
def capture_manifest() -> list:
|
||||
pages = list()
|
||||
after_id = 0
|
||||
with get_db_session() as session:
|
||||
while True:
|
||||
page = lib.build_manifest_page(session, after_id, limit=2)
|
||||
pages.append(page)
|
||||
if page['next_after_id'] is None:
|
||||
return pages
|
||||
after_id = page['next_after_id']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync(async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api,
|
||||
refresh_files, tag_factory, test_tags_config, events_fixture, await_switches):
|
||||
"""A secondary gets the primary's files, then its tags.yaml, and its FileGroups are tagged like the primary's."""
|
||||
write_primary_file(primary, 'docs/notes.txt', b'some notes', mtime=1_700_000_001.0)
|
||||
write_primary_file(primary, 'docs/more notes.txt', b'more notes', mtime=1_700_000_002.0)
|
||||
write_primary_file(primary, 'videos/channel/video.mp4', b'not really a video', mtime=1_700_000_003.0)
|
||||
write_primary_file(primary, 'videos/channel/video.info.json', b'{}', mtime=1_700_000_004.0)
|
||||
await refresh_files()
|
||||
|
||||
tag = await tag_factory('one')
|
||||
notes = test_session.query(FileGroup).filter_by(primary_path=primary.directory / 'docs/notes.txt').one()
|
||||
notes.add_tag(test_session, tag.id)
|
||||
test_session.commit()
|
||||
get_tags_config().dump_config()
|
||||
await await_switches()
|
||||
primary_tags = (primary.directory / 'config/tags.yaml').read_bytes()
|
||||
assert b'docs/notes.txt' in primary_tags
|
||||
|
||||
pages = capture_manifest()
|
||||
assert len(pages) >= 2, 'the manifest is paged'
|
||||
fake_primary_api(pages, primary.directory)
|
||||
|
||||
# The secondary is another machine with its own (empty) database.
|
||||
test_session.query(TagFile).delete()
|
||||
test_session.query(Tag).delete()
|
||||
test_session.query(FileGroup).delete()
|
||||
test_session.commit()
|
||||
|
||||
# This WROLPi becomes the secondary; it has its own (empty) tags.yaml.
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
(secondary_directory / 'config/tags.yaml').write_text('tags: {}\ntag_files: []\nversion: 1\n')
|
||||
|
||||
status = await lib.run_sync()
|
||||
|
||||
assert status['state'] == 'idle', status['last_error']
|
||||
assert status['files_failed'] == 0
|
||||
assert status['files_downloaded'] == 4
|
||||
assert status['last_success_at']
|
||||
for path in ('docs/notes.txt', 'docs/more notes.txt', 'videos/channel/video.mp4',
|
||||
'videos/channel/video.info.json'):
|
||||
copy = secondary_directory / path
|
||||
assert copy.read_bytes() == (primary.directory / path).read_bytes()
|
||||
assert copy.stat().st_mtime == (primary.directory / path).stat().st_mtime
|
||||
|
||||
# The secondary's own tags.yaml is kept in today's backup.
|
||||
backups = list((secondary_directory / 'config/backup').glob('tags-*.yaml'))
|
||||
assert len(backups) == 1 and b'docs/notes.txt' not in backups[0].read_bytes()
|
||||
|
||||
# The primary's tags.yaml is copied byte for byte, and imported.
|
||||
assert (secondary_directory / 'config/tags.yaml').read_bytes() == primary_tags
|
||||
assert 'tags.yaml' in status['configs_imported']
|
||||
with get_db_session() as session:
|
||||
copy = session.query(FileGroup).filter_by(primary_path=secondary_directory / 'docs/notes.txt').one()
|
||||
assert copy.tag_names == ['one']
|
||||
# Files of a group arrive together, and are grouped again on the secondary.
|
||||
video = session.query(FileGroup).filter_by(directory=secondary_directory / 'videos/channel').one()
|
||||
assert {pathlib.Path(i['path']).name for i in video.files} == {'video.mp4', 'video.info.json'}
|
||||
|
||||
events_fixture.assert_has_event('sync_completed')
|
||||
|
||||
# Nothing changed, so nothing is downloaded or imported again.
|
||||
primary.state['requests'].clear()
|
||||
status = await lib.run_sync()
|
||||
assert status['files_to_download'] == 0
|
||||
assert status['configs_imported'] == []
|
||||
assert not [i for i in primary.state['requests'] if not i[0].startswith('/media/config/')]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_replaced_file_keeps_its_links(async_client, primary, secondary_directory, make_secondary,
|
||||
fake_primary_api):
|
||||
"""The Tags Directory and playlists links of a file the sync replaces are links to the new version. A link to
|
||||
the old version would be the old version's last copy, which neither directory ever deletes."""
|
||||
changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0)
|
||||
same = write_primary_file(primary, 'docs/same.txt', b'unchanged')
|
||||
fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime), (same.path, same.size, same.mtime))],
|
||||
primary.directory)
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
# The secondary has an older version of one file, linked into a tag and a playlist.
|
||||
notes = secondary_directory / 'docs/notes.txt'
|
||||
unchanged = secondary_directory / 'docs/same.txt'
|
||||
notes.parent.mkdir(parents=True)
|
||||
notes.write_bytes(b'old notes')
|
||||
unchanged.write_bytes(b'unchanged')
|
||||
os.utime(unchanged, (same.mtime, same.mtime))
|
||||
tag_link = secondary_directory / 'tags/one/notes.txt'
|
||||
playlist_link = get_playlists_directory() / 'Reading/0001_notes.txt'
|
||||
unchanged_link = secondary_directory / 'tags/one/same.txt'
|
||||
# The only copy of a file that someone put in the Tags Directory themselves.
|
||||
stranger = secondary_directory / 'tags/one/stranger.txt'
|
||||
for source, link in ((notes, tag_link), (notes, playlist_link), (unchanged, unchanged_link)):
|
||||
link.parent.mkdir(parents=True, exist_ok=True)
|
||||
link.hardlink_to(source)
|
||||
stranger.write_bytes(b'old notes')
|
||||
|
||||
status = await lib.run_sync()
|
||||
|
||||
assert status['state'] == 'idle', status['last_error']
|
||||
assert status['files_downloaded'] == 1
|
||||
assert notes.read_bytes() == b'the new notes'
|
||||
for link in (tag_link, playlist_link):
|
||||
assert link.samefile(notes), f'{link} is still the old version'
|
||||
assert notes.stat().st_nlink == 3
|
||||
assert unchanged_link.samefile(unchanged)
|
||||
assert stranger.read_bytes() == b'old notes' and stranger.stat().st_nlink == 1
|
||||
assert not [i for i in secondary_directory.rglob('.*') if i.is_file()], 'no temporary file is left behind'
|
||||
|
||||
# The links are no longer wanted (the tag was removed, the playlist reordered): both directories can delete them.
|
||||
_delete_extra_tags_directory_paths(secondary_directory / 'tags', [unchanged_link, stranger])
|
||||
assert not tag_link.exists()
|
||||
assert _safe_unlink_managed(playlist_link)
|
||||
assert notes.read_bytes() == b'the new notes' and notes.stat().st_nlink == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_replaced_file_in_a_playlist_with_its_own_directory(
|
||||
async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api):
|
||||
"""A playlist may keep its links in any folder of the library; a replaced file's link there is moved too, and
|
||||
the folder's other files are left alone."""
|
||||
changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0)
|
||||
fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory)
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
notes = secondary_directory / 'docs/notes.txt'
|
||||
notes.parent.mkdir(parents=True)
|
||||
notes.write_bytes(b'old notes')
|
||||
directory = secondary_directory / 'my lists/Reading'
|
||||
(directory / 'deeper').mkdir(parents=True)
|
||||
link = directory / '0001_notes.txt'
|
||||
link.hardlink_to(notes)
|
||||
# A hard link of the file that the playlist did not make: not its name, or not directly in its folder.
|
||||
renamed = directory / 'my copy of the notes.txt'
|
||||
deeper = directory / 'deeper/0001_notes.txt'
|
||||
renamed.hardlink_to(notes)
|
||||
deeper.hardlink_to(notes)
|
||||
test_session.add(Collection(name='Reading', kind='playlist', directory=directory))
|
||||
test_session.commit()
|
||||
|
||||
status = await lib.run_sync()
|
||||
|
||||
assert status['state'] == 'idle', status['last_error']
|
||||
assert link.samefile(notes) and link.read_bytes() == b'the new notes'
|
||||
assert renamed.read_bytes() == b'old notes' and deeper.read_bytes() == b'old notes'
|
||||
assert _safe_unlink_managed(link)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_replaced_file_with_a_hard_linked_twin(async_client, primary, secondary_directory,
|
||||
make_secondary, fake_primary_api):
|
||||
"""Two library files may be hard links of each other. When one of them changes, only links named for it move;
|
||||
its twin's link is still its twin."""
|
||||
changed = write_primary_file(primary, 'docs/a.txt', b'the new a', mtime=1_700_000_500.0)
|
||||
fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory)
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
a, b = secondary_directory / 'docs/a.txt', secondary_directory / 'docs/b.txt'
|
||||
a.parent.mkdir(parents=True)
|
||||
a.write_bytes(b'shared')
|
||||
b.hardlink_to(a)
|
||||
a_link, b_link = secondary_directory / 'tags/one/a.txt', secondary_directory / 'tags/one/b.txt'
|
||||
a_link.parent.mkdir(parents=True)
|
||||
a_link.hardlink_to(a)
|
||||
b_link.hardlink_to(b)
|
||||
|
||||
status = await lib.run_sync()
|
||||
|
||||
assert status['state'] == 'idle', status['last_error']
|
||||
assert a.read_bytes() == b'the new a' and a_link.samefile(a)
|
||||
assert b.read_bytes() == b'shared' and b_link.samefile(b)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_interrupted_before_a_file_is_replaced(async_client, primary, secondary_directory,
|
||||
make_secondary, fake_primary_api, monkeypatch):
|
||||
"""A sync that stops between downloading a new version and putting it in place has left no link on the old
|
||||
version once a later sync finishes the job, although that sync downloads nothing."""
|
||||
changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0)
|
||||
fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory)
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
notes = secondary_directory / 'docs/notes.txt'
|
||||
notes.parent.mkdir(parents=True)
|
||||
notes.write_bytes(b'old notes')
|
||||
link = secondary_directory / 'tags/one/notes.txt'
|
||||
link.parent.mkdir(parents=True)
|
||||
link.hardlink_to(notes)
|
||||
|
||||
replace = os.replace
|
||||
|
||||
def power_cut(source, destination):
|
||||
if pathlib.Path(destination) == notes:
|
||||
raise OSError('the power went out')
|
||||
return replace(source, destination)
|
||||
|
||||
monkeypatch.setattr(lib.os, 'replace', power_cut)
|
||||
status = await lib.run_sync()
|
||||
assert status['files_failed'] == 1
|
||||
assert notes.read_bytes() == b'old notes', 'the file was not replaced'
|
||||
monkeypatch.setattr(lib.os, 'replace', replace)
|
||||
|
||||
primary.state['requests'].clear()
|
||||
status = await lib.run_sync()
|
||||
|
||||
assert status['state'] == 'idle', status['last_error']
|
||||
assert status['files_downloaded'] == 1 and status['bytes_downloaded'] == 0
|
||||
assert notes.read_bytes() == b'the new notes'
|
||||
assert link.samefile(notes) and notes.stat().st_nlink == 2
|
||||
assert not [i for i in secondary_directory.rglob('.*') if i.is_file()]
|
||||
|
||||
|
||||
def test_move_links_of_a_long_name(test_session, test_directory):
|
||||
"""A link whose name is as long as a file name may be is still moved."""
|
||||
name = 'n' * 240 + '.txt'
|
||||
old, new = test_directory / 'docs' / name, test_directory / 'docs/new version'
|
||||
old.parent.mkdir()
|
||||
old.write_bytes(b'old')
|
||||
new.write_bytes(b'new')
|
||||
link = get_playlists_directory() / 'Reading' / f'0001_{name}'
|
||||
link.parent.mkdir(parents=True)
|
||||
link.hardlink_to(old)
|
||||
|
||||
assert lib.find_links([old]) == {old: [link]}
|
||||
lib.move_links([link], old, new)
|
||||
|
||||
assert link.samefile(new)
|
||||
assert [i.name for i in link.parent.iterdir()] == [link.name]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_refuses_unsafe_paths(async_client, primary, secondary_directory, make_secondary,
|
||||
fake_primary_api):
|
||||
"""A manifest cannot make a secondary write outside its library, or over its configs and database."""
|
||||
good = write_primary_file(primary, 'good.txt', b'good')
|
||||
write_primary_file(primary, 'config/wrolpi.db', b'not a database')
|
||||
pages = [manifest_page(
|
||||
(good.path, good.size, good.mtime),
|
||||
('config/wrolpi.db', 14, 1_700_000_000.0),
|
||||
('../escape.txt', 4, 1_700_000_000.0),
|
||||
('tags/one/good.txt', 4, 1_700_000_000.0),
|
||||
)]
|
||||
fake_primary_api(pages, primary.directory)
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
status = await lib.run_sync()
|
||||
|
||||
assert status['files_failed'] == 3
|
||||
assert (secondary_directory / 'good.txt').read_bytes() == b'good'
|
||||
assert not (secondary_directory / 'config/wrolpi.db').exists()
|
||||
assert not (secondary_directory.parent / 'escape.txt').exists()
|
||||
assert not (secondary_directory / 'tags').exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_excluded_directories(async_client, primary, secondary_directory, make_secondary,
|
||||
fake_primary_api):
|
||||
"""A secondary never downloads files inside its excluded folders, and keeps what it already has in them."""
|
||||
wanted = [write_primary_file(primary, i, b'wanted') for i in ('Movies2/a.mp4', 'docs/Movies/b.txt', 'c.txt')]
|
||||
excluded = [write_primary_file(primary, i, b'excluded') for i in ('Movies/a.mp4', 'Movies/deep/b.mp4',
|
||||
'Movies [HD]/c.mp4')]
|
||||
pages = [manifest_page(*((i.path, i.size, i.mtime) for i in wanted + excluded))]
|
||||
fake_primary_api(pages, primary.directory)
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
get_sync_config()._config['sync_excluded_directories'] = ['Movies', 'Movies [HD]']
|
||||
# Copied before the folder was excluded.
|
||||
(secondary_directory / 'Movies').mkdir()
|
||||
(secondary_directory / 'Movies/old.mp4').write_bytes(b'old')
|
||||
|
||||
status = await lib.run_sync()
|
||||
|
||||
assert status['last_error'] is None
|
||||
assert status['files_excluded'] == 3
|
||||
assert status['files_downloaded'] == 3
|
||||
for file in wanted:
|
||||
assert (secondary_directory / file.path).read_bytes() == b'wanted'
|
||||
for file in excluded:
|
||||
assert not (secondary_directory / file.path).exists()
|
||||
assert (secondary_directory / 'Movies/old.mp4').read_bytes() == b'old', 'nothing is deleted'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('values, expected', [
|
||||
([], []),
|
||||
(['Movies'], ['Movies']),
|
||||
(['/Movies/', ' Movies ', 'Movies//'], ['Movies']),
|
||||
(['videos/Movies', 'videos'], ['videos']),
|
||||
(['Movies', 'Movies2', 'Movies/deep'], ['Movies', 'Movies2']),
|
||||
(['a//b'], ['a/b']),
|
||||
(['Movies [HD]'], ['Movies [HD]']),
|
||||
])
|
||||
def test_normalize_excluded_directories(values, expected):
|
||||
assert lib.normalize_excluded_directories(values) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize('value', ['', '/', '..', 'a/../b', '../a', 'a/./b', '.hidden', 'a/.hidden', 'config',
|
||||
'config/x', 'tags', 'Config', 'TAGS/x', 'a\\b', None])
|
||||
def test_normalize_excluded_directories_rejects(value):
|
||||
with pytest.raises(InvalidConfig):
|
||||
lib.normalize_excluded_directories([value])
|
||||
|
||||
|
||||
def test_is_excluded():
|
||||
assert lib.is_excluded('Movies/a.mp4', {'Movies'})
|
||||
assert lib.is_excluded('Movies', {'Movies'})
|
||||
assert not lib.is_excluded('Movies2/a.mp4', {'Movies'})
|
||||
assert not lib.is_excluded('docs/Movies/a.mp4', {'Movies'})
|
||||
assert lib.is_excluded('docs/Movies/a.mp4', {'docs/Movies'})
|
||||
|
||||
|
||||
def test_list_local_directories(test_directory):
|
||||
for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'zims', 'lost+found', 'Tags'):
|
||||
try:
|
||||
(test_directory / i).mkdir(parents=True)
|
||||
except FileExistsError:
|
||||
pass # 'Tags' is 'tags' on a filesystem that ignores case.
|
||||
(test_directory / 'file.txt').write_text('a file')
|
||||
(test_directory / 'link').symlink_to(test_directory / 'Movies')
|
||||
|
||||
assert lib.list_local_directories('') == [
|
||||
dict(name='Movies', path='Movies'), dict(name='videos', path='videos'), dict(name='zims', path='zims')]
|
||||
assert lib.list_local_directories('/videos/') == [dict(name='channel', path='videos/channel')]
|
||||
for path in ('../', 'videos/../..', 'file.txt', 'missing', 'link'):
|
||||
with pytest.raises(InvalidConfig):
|
||||
lib.list_local_directories(path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_primary_directories(primary, secondary_directory, make_secondary):
|
||||
for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'lost+found'):
|
||||
(primary.directory / i).mkdir(parents=True)
|
||||
(primary.directory / 'file.txt').write_text('a file')
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
assert await lib.list_primary_directories('') == [
|
||||
dict(name='Movies', path='Movies'), dict(name='videos', path='videos')]
|
||||
assert await lib.list_primary_directories('videos') == [dict(name='channel', path='videos/channel')]
|
||||
with pytest.raises(InvalidConfig):
|
||||
await lib.list_primary_directories('../etc')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_only_on_a_secondary(async_client, test_wrolpi_config, fake_primary_api):
|
||||
status = await lib.run_sync()
|
||||
assert status['state'] == 'idle'
|
||||
assert status['last_skip_reason'] == 'This WROLPi is not a secondary'
|
||||
assert status['started_at'] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_refuses_a_secondary_primary(async_client, primary, secondary_directory, make_secondary,
|
||||
fake_primary_api):
|
||||
"""Secondaries do not chain; a secondary's library may be stale."""
|
||||
fake_primary_api([manifest_page()], primary.directory, role='secondary')
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
status = await lib.run_sync()
|
||||
assert status['state'] == 'error'
|
||||
assert 'is not a primary' in status['last_error']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_sync_cancel(async_client, primary, secondary_directory, make_secondary, fake_primary_api):
|
||||
"""A cancelled sync stops between pages; the files of finished pages stay."""
|
||||
first = write_primary_file(primary, 'first.txt', b'first')
|
||||
second = write_primary_file(primary, 'second.txt', b'second')
|
||||
pages = [manifest_page((first.path, first.size, first.mtime), next_after_id=1, max_id=2),
|
||||
manifest_page((second.path, second.size, second.mtime), max_id=2)]
|
||||
fake_primary_api(pages, primary.directory)
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
original = lib._refresh
|
||||
|
||||
async def refresh_then_cancel(paths):
|
||||
await original(paths)
|
||||
lib.request_sync_cancel()
|
||||
|
||||
lib._refresh = refresh_then_cancel
|
||||
try:
|
||||
status = await lib.run_sync()
|
||||
finally:
|
||||
lib._refresh = original
|
||||
|
||||
assert status['last_error'] == 'The sync was cancelled'
|
||||
assert status['state'] == 'idle'
|
||||
assert (secondary_directory / 'first.txt').exists()
|
||||
assert not (secondary_directory / 'second.txt').exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_configs_wrolpi_yaml(async_client, primary, secondary_directory, make_secondary):
|
||||
"""Only the library settings of the primary's wrolpi.yaml are applied; the secondary keeps its own role, hotspot,
|
||||
and the like."""
|
||||
primary_config = dict(
|
||||
videos_destination='my videos/%(channel_name)s',
|
||||
ignored_directories=['config', 'tags', str(primary.directory / 'private'), '/elsewhere/directory'],
|
||||
hotspot_ssid='primary hotspot',
|
||||
role='primary',
|
||||
nav_color='red',
|
||||
version=12,
|
||||
)
|
||||
(primary.directory / 'config').mkdir(exist_ok=True)
|
||||
(primary.directory / 'config/wrolpi.yaml').write_text(yaml.dump(primary_config))
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
async with lib.PrimaryClient(primary.url) as client:
|
||||
await lib.sync_configs(client, str(primary.directory), reimport=False)
|
||||
|
||||
config = get_wrolpi_config()
|
||||
assert config.videos_destination == 'my videos/%(channel_name)s'
|
||||
assert config.ignored_directories == ['config', 'tags', 'private']
|
||||
assert get_sync_config().role == 'secondary', 'the role is in sync.yaml, which is never synced'
|
||||
assert config.hotspot_ssid == 'WROLPi'
|
||||
assert config.nav_color == 'violet'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_configs_inventories(async_client, primary, secondary_directory, make_secondary):
|
||||
"""Inventories are a directory of configs; each one is copied."""
|
||||
from modules.inventory.common import get_inventory_configs
|
||||
|
||||
inventory_directory = primary.directory / 'config/inventory'
|
||||
inventory_directory.mkdir(parents=True)
|
||||
pantry = dict(version=3, slug='pantry', name='Pantry', type='food', fields=[], items=[])
|
||||
(inventory_directory / 'pantry.yaml').write_text(yaml.dump(pantry))
|
||||
(inventory_directory / '.hidden.yaml').write_text('ignored')
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
async with lib.PrimaryClient(primary.url) as client:
|
||||
changed = await lib._sync_inventory_directory(client)
|
||||
|
||||
assert changed
|
||||
copy = get_inventory_configs().get_directory() / 'pantry.yaml'
|
||||
assert yaml.safe_load(copy.read_text())['name'] == 'Pantry'
|
||||
assert not (copy.parent / '.hidden.yaml').exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_tick(async_client, test_wrolpi_config, make_secondary, monkeypatch):
|
||||
"""A requested sync runs; WROL Mode drops the request."""
|
||||
calls = list()
|
||||
|
||||
async def fake_run_sync(force=False):
|
||||
calls.append(force)
|
||||
|
||||
monkeypatch.setattr(lib, 'run_sync', fake_run_sync)
|
||||
|
||||
await lib.sync_tick()
|
||||
assert calls == [], 'a primary never syncs'
|
||||
|
||||
make_secondary()
|
||||
await lib.sync_tick()
|
||||
assert calls == [], 'nothing requested, no window'
|
||||
|
||||
lib.request_sync(force=True)
|
||||
await lib.sync_tick()
|
||||
assert calls == [True]
|
||||
assert get_sync_status()['requested'] is False
|
||||
|
||||
get_wrolpi_config()._config['wrol_mode'] = True
|
||||
lib.request_sync()
|
||||
await lib.sync_tick()
|
||||
assert calls == [True]
|
||||
assert get_sync_status()['requested'] is False
|
||||
assert get_sync_status()['last_skip_reason'] == 'WROL Mode is enabled'
|
||||
|
||||
|
||||
def test_request_sync_cancel(async_client):
|
||||
update_sync_status(state='running')
|
||||
lib.request_sync_cancel()
|
||||
assert get_sync_status()['cancel_requested'] is True
|
||||
|
||||
update_sync_status(state='idle', cancel_requested=False, requested=True)
|
||||
lib.request_sync_cancel()
|
||||
assert get_sync_status()['requested'] is False
|
||||
assert get_sync_status()['cancel_requested'] is False
|
||||
|
||||
|
||||
def test_partial_path():
|
||||
assert lib.partial_path(pathlib.Path('/a/b/video.mp4')) == pathlib.Path('/a/b/.video.mp4.partial')
|
||||
116
wrolpi/sync/test/test_lib.py
Normal file
116
wrolpi/sync/test/test_lib.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
"""What is left of the API's side of a secondary's sync: its settings, and the primary's folders it may exclude.
|
||||
The sync itself is the Controller's (controller/test/test_sync.py)."""
|
||||
from collections import namedtuple
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.common import set_test_media_directory
|
||||
from wrolpi.errors import InvalidConfig
|
||||
from wrolpi.sync import lib
|
||||
|
||||
@pytest.mark.parametrize('path, allowed', [
|
||||
('videos/channel/video.mp4', True),
|
||||
('a.txt', True),
|
||||
('../escape.txt', False),
|
||||
('videos/../../escape.txt', False),
|
||||
('/etc/passwd', False),
|
||||
('videos/.hidden/video.mp4', False),
|
||||
('.hidden.txt', False),
|
||||
('config/wrolpi.db', False),
|
||||
('config/tags.yaml', False),
|
||||
('tags/one/video.mp4', False),
|
||||
# A drive that ignores case (exFAT, NTFS, macOS) would write these into the real config and tags folders.
|
||||
('Config/wrolpi.db', False),
|
||||
('CONFIG/tags.yaml', False),
|
||||
('Tags/one/video.mp4', False),
|
||||
('videos\\video.mp4', False),
|
||||
('', False),
|
||||
])
|
||||
def test_resolve_destination(test_directory, path, allowed):
|
||||
destination = lib.resolve_destination(test_directory, path)
|
||||
if allowed:
|
||||
assert destination == test_directory / path
|
||||
else:
|
||||
assert destination is None
|
||||
|
||||
|
||||
def test_resolve_destination_refuses_symlinks(test_directory):
|
||||
"""A sync never writes through a symlink; it could point outside the media directory."""
|
||||
outside = test_directory / 'outside'
|
||||
outside.mkdir()
|
||||
(test_directory / 'videos').symlink_to(outside)
|
||||
assert lib.resolve_destination(test_directory, 'videos/video.mp4') is None
|
||||
|
||||
(test_directory / 'docs').mkdir()
|
||||
(test_directory / 'docs/link.txt').symlink_to(outside / 'target.txt')
|
||||
assert lib.resolve_destination(test_directory, 'docs/link.txt') is None
|
||||
|
||||
|
||||
Primary = namedtuple('Primary', 'directory url state')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def primary(test_directory, secondary_directory, primary_media_server) -> Primary:
|
||||
"""A primary whose media is `test_directory`, served like Caddy."""
|
||||
url, state = primary_media_server(test_directory)
|
||||
return Primary(test_directory, url, state)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('values, expected', [
|
||||
([], []),
|
||||
(['Movies'], ['Movies']),
|
||||
(['/Movies/', ' Movies ', 'Movies//'], ['Movies']),
|
||||
(['videos/Movies', 'videos'], ['videos']),
|
||||
(['Movies', 'Movies2', 'Movies/deep'], ['Movies', 'Movies2']),
|
||||
(['a//b'], ['a/b']),
|
||||
(['Movies [HD]'], ['Movies [HD]']),
|
||||
])
|
||||
def test_normalize_excluded_directories(values, expected):
|
||||
assert lib.normalize_excluded_directories(values) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize('value', ['', '/', '..', 'a/../b', '../a', 'a/./b', '.hidden', 'a/.hidden', 'config',
|
||||
'config/x', 'tags', 'Config', 'TAGS/x', 'a\\b', None])
|
||||
def test_normalize_excluded_directories_rejects(value):
|
||||
with pytest.raises(InvalidConfig):
|
||||
lib.normalize_excluded_directories([value])
|
||||
|
||||
|
||||
def test_is_excluded():
|
||||
assert lib.is_excluded('Movies/a.mp4', {'Movies'})
|
||||
assert lib.is_excluded('Movies', {'Movies'})
|
||||
assert not lib.is_excluded('Movies2/a.mp4', {'Movies'})
|
||||
assert not lib.is_excluded('docs/Movies/a.mp4', {'Movies'})
|
||||
assert lib.is_excluded('docs/Movies/a.mp4', {'docs/Movies'})
|
||||
|
||||
|
||||
def test_list_local_directories(test_directory):
|
||||
for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'zims', 'lost+found', 'Tags'):
|
||||
try:
|
||||
(test_directory / i).mkdir(parents=True)
|
||||
except FileExistsError:
|
||||
pass # 'Tags' is 'tags' on a filesystem that ignores case.
|
||||
(test_directory / 'file.txt').write_text('a file')
|
||||
(test_directory / 'link').symlink_to(test_directory / 'Movies')
|
||||
|
||||
assert lib.list_local_directories('') == [
|
||||
dict(name='Movies', path='Movies'), dict(name='videos', path='videos'), dict(name='zims', path='zims')]
|
||||
assert lib.list_local_directories('/videos/') == [dict(name='channel', path='videos/channel')]
|
||||
for path in ('../', 'videos/../..', 'file.txt', 'missing', 'link'):
|
||||
with pytest.raises(InvalidConfig):
|
||||
lib.list_local_directories(path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_primary_directories(primary, secondary_directory, make_secondary):
|
||||
for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'lost+found'):
|
||||
(primary.directory / i).mkdir(parents=True)
|
||||
(primary.directory / 'file.txt').write_text('a file')
|
||||
set_test_media_directory(secondary_directory)
|
||||
make_secondary(primary.url)
|
||||
|
||||
assert await lib.list_primary_directories('') == [
|
||||
dict(name='Movies', path='Movies'), dict(name='videos', path='videos')]
|
||||
assert await lib.list_primary_directories('videos') == [dict(name='channel', path='videos/channel')]
|
||||
with pytest.raises(InvalidConfig):
|
||||
await lib.list_primary_directories('../etc')
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.sync import lib
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_pages(async_client, test_session, test_directory, make_files_structure, refresh_files):
|
||||
"""The manifest lists every file of every FileGroup, a page of FileGroups at a time, in id order."""
|
||||
make_files_structure({f'docs/{i}.txt': f'file {i}' for i in range(7)})
|
||||
await refresh_files()
|
||||
|
||||
pages = list()
|
||||
after_id = 0
|
||||
while after_id is not None:
|
||||
page = lib.build_manifest_page(test_session, after_id, limit=3)
|
||||
pages.append(page)
|
||||
after_id = page['next_after_id']
|
||||
|
||||
assert [len(i['groups']) for i in pages] == [3, 3, 1]
|
||||
ids = [g['id'] for page in pages for g in page['groups']]
|
||||
assert ids == sorted(ids)
|
||||
assert pages[0]['max_id'] == max(ids)
|
||||
|
||||
files = {p['path']: p for page in pages for g in page['groups'] for p in g['paths']}
|
||||
assert set(files) == {f'docs/{i}.txt' for i in range(7)}
|
||||
stat = (test_directory / 'docs/3.txt').stat()
|
||||
assert files['docs/3.txt']['size'] == stat.st_size
|
||||
assert files['docs/3.txt']['mtime'] == stat.st_mtime
|
||||
|
||||
|
||||
def test_manifest_skips(test_session, test_directory):
|
||||
"""Files that are still downloading, hidden, missing, or outside the library are not listed."""
|
||||
directory = test_directory / 'videos'
|
||||
directory.mkdir()
|
||||
for name in ('video.mp4', 'video.mp4.part', 'video.f137.mp4', 'video.mp4.aria2', '.video.mp4.partial',
|
||||
'video.en.vtt'):
|
||||
(directory / name).write_text(name)
|
||||
outside = test_directory.parent / f'{test_directory.name}-outside.txt'
|
||||
outside.write_text('outside')
|
||||
try:
|
||||
group = FileGroup.from_paths(test_session, directory / 'video.mp4', directory / 'video.en.vtt')
|
||||
group.files = [
|
||||
*group.files,
|
||||
dict(path='video.mp4.part'), dict(path='video.f137.mp4'), dict(path='video.mp4.aria2'),
|
||||
dict(path='.video.mp4.partial'), dict(path='deleted.mp4'),
|
||||
# Older rows stored absolute paths.
|
||||
dict(path=str(directory / 'video.en.vtt')),
|
||||
dict(path=str(outside)),
|
||||
{},
|
||||
]
|
||||
test_session.commit()
|
||||
|
||||
page = lib.build_manifest_page(test_session)
|
||||
finally:
|
||||
os.remove(outside)
|
||||
|
||||
paths = [p['path'] for g in page['groups'] for p in g['paths']]
|
||||
assert sorted(set(paths)) == ['videos/video.en.vtt', 'videos/video.mp4']
|
||||
assert page['next_after_id'] is None
|
||||
|
||||
|
||||
def test_manifest_empty(test_session, test_directory):
|
||||
assert lib.build_manifest_page(test_session) == dict(groups=[], next_after_id=None, max_id=0,
|
||||
limit=lib.MANIFEST_PAGE_LIMIT)
|
||||
72
wrolpi/sync/test/test_notify.py
Normal file
72
wrolpi/sync/test/test_notify.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import pytest
|
||||
import yaml
|
||||
|
||||
from wrolpi.sync import notify
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def state(test_directory):
|
||||
"""Write the Controller's record of the last sync."""
|
||||
|
||||
def write(**fields):
|
||||
record = dict(state='idle', finished_at='2026-11-19T03:00:00+00:00', last_error=None, files_transferred=0,
|
||||
files_deleted=0, warnings=[])
|
||||
record.update(fields)
|
||||
(test_directory / notify.STATE_FILE).write_text(yaml.safe_dump(record))
|
||||
|
||||
return write
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fields, expected', [
|
||||
(dict(files_transferred=12, files_deleted=3),
|
||||
('completed', 'Synced 12 files from the primary; 3 moved to the trash')),
|
||||
(dict(files_transferred=1), ('completed', 'Synced 1 files from the primary')),
|
||||
(dict(warnings=['Some files changed on the primary']),
|
||||
('completed', 'Synced 0 files from the primary; Some files changed on the primary')),
|
||||
(dict(state='error', last_error='The primary did not answer'),
|
||||
('failed', 'Sync from the primary failed: The primary did not answer')),
|
||||
# Nothing changed: within the window that is every 15 minutes.
|
||||
(dict(), None),
|
||||
(dict(last_error='The sync was cancelled', files_transferred=4), None),
|
||||
(dict(last_error='WROL Mode was enabled'), None),
|
||||
])
|
||||
def test_sync_message(fields, expected):
|
||||
record = dict(state='idle', last_error=None, files_transferred=0, files_deleted=0, warnings=[])
|
||||
record.update(fields)
|
||||
assert notify.sync_message(record) == expected
|
||||
|
||||
|
||||
def test_each_sync_is_announced_once(async_client, test_directory, test_wrolpi_config, make_secondary, state,
|
||||
events_fixture):
|
||||
"""The API is restarted by every sync, so what it announced is kept in a file of its own."""
|
||||
make_secondary('https://primary.example')
|
||||
state(files_transferred=5)
|
||||
notify.notify_tick()
|
||||
events_fixture.assert_has_event('sync_completed')
|
||||
assert yaml.safe_load((test_directory / notify.ANNOUNCED_FILE).read_text()) == dict(
|
||||
finished_at='2026-11-19T03:00:00+00:00')
|
||||
|
||||
# Again, and after a restart: the same sync is not announced twice.
|
||||
notify.notify_tick()
|
||||
assert len(events_fixture.get_by_name('sync_completed')) == 1
|
||||
|
||||
state(state='error', last_error='The primary did not answer', finished_at='2026-11-19T03:20:00+00:00')
|
||||
notify.notify_tick()
|
||||
events_fixture.assert_has_event('sync_failed', 'Sync from the primary failed: The primary did not answer')
|
||||
|
||||
# A sync that changed nothing is recorded as seen, and not announced.
|
||||
state(finished_at='2026-11-19T03:40:00+00:00')
|
||||
notify.notify_tick()
|
||||
assert len(events_fixture.get_by_name('sync_completed')) == 1
|
||||
assert yaml.safe_load((test_directory / notify.ANNOUNCED_FILE).read_text())['finished_at'] == \
|
||||
'2026-11-19T03:40:00+00:00'
|
||||
|
||||
|
||||
def test_not_while_running_and_not_on_a_primary(async_client, test_directory, test_wrolpi_config, make_secondary,
|
||||
state):
|
||||
state(state='running', finished_at=None)
|
||||
notify.notify_tick()
|
||||
assert not (test_directory / notify.ANNOUNCED_FILE).exists(), 'on a primary nothing is read'
|
||||
make_secondary('https://primary.example')
|
||||
notify.notify_tick()
|
||||
assert not (test_directory / notify.ANNOUNCED_FILE).exists(), 'a running sync has nothing to announce'
|
||||
|
|
@ -1,16 +1,14 @@
|
|||
"""A secondary WROLPi never saves the configs it copies from its primary, and never downloads."""
|
||||
from datetime import datetime, time, timezone
|
||||
from datetime import time
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from wrolpi.common import get_wrolpi_config, SYNCED_CONFIG_FILE_NAMES, get_all_configs
|
||||
from wrolpi.config_version import IMPORT, SAVE, get_config_version, set_config_version
|
||||
from wrolpi.config_version import IMPORT, SAVE, set_config_version
|
||||
from wrolpi.dates import is_within_daily_window
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.sync import lib
|
||||
from wrolpi.sync.config import get_sync_config
|
||||
from wrolpi.tags import get_tags_config
|
||||
|
||||
|
||||
|
|
@ -87,19 +85,3 @@ def test_secondary_never_downloads(async_client, test_wrolpi_config, make_second
|
|||
])
|
||||
def test_is_within_daily_window(start, end, current, expected):
|
||||
assert is_within_daily_window(start, end, current) is expected
|
||||
|
||||
|
||||
def test_sync_is_due(test_wrolpi_config):
|
||||
config = get_sync_config()
|
||||
local_now = datetime(2026, 11, 5, 2, 0, tzinfo=timezone.utc)
|
||||
never = dict(finished_at=None)
|
||||
assert not lib.sync_is_due(never, local_now), 'no window is "Sync now" only'
|
||||
|
||||
config._config.update(window_start='01:00', window_end='05:00')
|
||||
assert lib.sync_is_due(never, local_now)
|
||||
assert not lib.sync_is_due(never, local_now.replace(hour=6)), 'outside the window'
|
||||
|
||||
recent = dict(finished_at=local_now.replace(hour=1, minute=50).isoformat())
|
||||
assert not lib.sync_is_due(recent, local_now), 'synced 10 minutes ago'
|
||||
earlier = dict(finished_at=local_now.replace(hour=1, minute=40).isoformat())
|
||||
assert lib.sync_is_due(earlier, local_now), 'synced 20 minutes ago'
|
||||
|
|
|
|||
|
|
@ -219,12 +219,11 @@ async def test_downloads_summary_fixture_matches_the_download_manager(async_clie
|
|||
|
||||
|
||||
@pytest.mark.parametrize('fixture_name, seed_name, response_name', [
|
||||
('syncStatusFixture', 'default_sync_status', 'SyncStatusResponse'),
|
||||
('backupStatusFixture', 'default_backup_status', 'BackupStatusResponse'),
|
||||
])
|
||||
def test_sync_status_fixture_matches_the_seed(fixture_name, seed_name, response_name):
|
||||
"""statusFixture's `sync` and `backup` are seeded by wrolpi.sync.status and declared in wrolpi.schema; the
|
||||
fixture, the seed and the declaration must name the same keys."""
|
||||
"""statusFixture's `backup` is seeded by wrolpi.sync.status and declared in wrolpi.schema; the fixture, the
|
||||
seed and the declaration must name the same keys."""
|
||||
from dataclasses import fields
|
||||
from wrolpi import schema
|
||||
from wrolpi.sync import status
|
||||
|
|
@ -271,11 +270,11 @@ async def test_which_fixtures_the_spec_can_vouch_for(async_client):
|
|||
|
||||
source = TEST_FIXTURES_JS.read_text()
|
||||
api_fixtures = set(re.findall(r'^export const ([a-zA-Z]+Fixture) =', source, re.M))
|
||||
# domainsFixture is a list built from domainFixture, syncStatusFixture and backupStatusFixture are
|
||||
# statusFixture's `sync` and `backup` (test_sync_status_fixture_matches_the_seed holds them), and the *Context* fixtures are held to
|
||||
# their React contexts by the frontend's own test-fixtures.test.js. controllerSyncStatusFixture is the
|
||||
# Controller's, not this API's: controller/test/test_sync.py holds it to the Controller's sync status.
|
||||
api_fixtures -= {'domainsFixture', 'syncStatusFixture', 'backupStatusFixture', 'controllerSyncStatusFixture'}
|
||||
# domainsFixture is a list built from domainFixture, backupStatusFixture is statusFixture's `backup`
|
||||
# (test_sync_status_fixture_matches_the_seed holds it), and the *Context* fixtures are held to their React
|
||||
# contexts by the frontend's own test-fixtures.test.js. controllerSyncStatusFixture is the Controller's, not
|
||||
# this API's: controller/test/test_sync.py holds it to the Controller's sync status.
|
||||
api_fixtures -= {'domainsFixture', 'backupStatusFixture', 'controllerSyncStatusFixture'}
|
||||
api_fixtures = {n for n in api_fixtures if 'Context' not in n}
|
||||
|
||||
assert api_fixtures == set(checked) | set(unchecked), (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue