diff --git a/alembic/versions/a1b2c3d4e5f6_add_collection_file_format.py b/alembic/versions/a1b2c3d4e5f6_add_collection_file_format.py new file mode 100644 index 00000000..b49491de --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_add_collection_file_format.py @@ -0,0 +1,48 @@ +"""add_collection_file_format + +Revision ID: a1b2c3d4e5f6 +Revises: 76add63ed225 +Create Date: 2026-01-23 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a1b2c3d4e5f6' +down_revision = '76add63ed225' +branch_labels = None +depends_on = None + + +def upgrade(): + # Add file_format column to collection table + # This stores the file_name_format used when files were last organized + op.add_column('collection', sa.Column('file_format', sa.String(), nullable=True)) + + # Set default values from current configs so existing collections don't need reorganization + from modules.archive.lib import get_archive_downloader_config + from modules.videos.lib import get_videos_downloader_config + + connection = op.get_bind() + + # Update domain collections with archive config's file_name_format + archive_config = get_archive_downloader_config() + if archive_config.file_name_format: + connection.execute( + sa.text("UPDATE collection SET file_format = :format WHERE kind = 'domain'"), + {'format': archive_config.file_name_format} + ) + + # Update channel collections with video config's file_name_format + video_config = get_videos_downloader_config() + if video_config.file_name_format: + connection.execute( + sa.text("UPDATE collection SET file_format = :format WHERE kind = 'channel'"), + {'format': video_config.file_name_format} + ) + + +def downgrade(): + op.drop_column('collection', 'file_format') diff --git a/alembic/versions/b2b9700f5048_fix_corrupted_media_paths.py b/alembic/versions/b2b9700f5048_fix_corrupted_media_paths.py new file mode 100644 index 00000000..47fe71b3 --- /dev/null +++ b/alembic/versions/b2b9700f5048_fix_corrupted_media_paths.py @@ -0,0 +1,189 @@ +"""Fix media paths incorrectly resolved against /opt/wrolpi. + +MediaPathType.process_bind_param() was calling .absolute() on relative paths, +which resolves against the current working directory (cwd) instead of the +media directory. In production, cwd is /opt/wrolpi, so paths like +'archive/spiegel.de' became '/opt/wrolpi/archive/spiegel.de' instead of +'/media/wrolpi/archive/spiegel.de'. + +This migration fixes any corrupted paths in the database and saves the +corrected paths to config files. + +Revision ID: b2b9700f5048 +Revises: a1b2c3d4e5f6 +Create Date: 2026-01-27 +""" +import os +import pathlib + +import yaml +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'b2b9700f5048' +down_revision = 'a1b2c3d4e5f6' +branch_labels = None +depends_on = None + + +def get_relative_path(absolute_path: str, media_directory: str) -> str: + """Convert absolute path to relative path under media directory.""" + if absolute_path and absolute_path.startswith(media_directory): + return absolute_path[len(media_directory):].lstrip('/') + return absolute_path + + +def save_config_file(config: dict, file_path: pathlib.Path, width: int = 120): + """Write config dict to YAML file.""" + file_path.parent.mkdir(parents=True, exist_ok=True) + with file_path.open('wt') as fh: + yaml.dump(config, fh, width=width, sort_keys=True) + fh.flush() + os.fsync(fh.fileno()) + + +def upgrade(): + connection = op.get_bind() + + # Fix collection.directory + result = connection.execute(sa.text(""" + UPDATE collection + SET directory = REPLACE(directory, '/opt/wrolpi/', '/media/wrolpi/') + WHERE directory LIKE '/opt/wrolpi/%' + """)) + if result.rowcount: + print(f"Fixed {result.rowcount} collection directory paths") + + # Fix file_group.directory + result = connection.execute(sa.text(""" + UPDATE file_group + SET directory = REPLACE(directory, '/opt/wrolpi/', '/media/wrolpi/') + WHERE directory LIKE '/opt/wrolpi/%' + """)) + if result.rowcount: + print(f"Fixed {result.rowcount} file_group directory paths") + + # Fix file_group.primary_path + result = connection.execute(sa.text(""" + UPDATE file_group + SET primary_path = REPLACE(primary_path, '/opt/wrolpi/', '/media/wrolpi/') + WHERE primary_path LIKE '/opt/wrolpi/%' + """)) + if result.rowcount: + print(f"Fixed {result.rowcount} file_group primary_path paths") + + # Fix download.destination + result = connection.execute(sa.text(""" + UPDATE download + SET destination = REPLACE(destination, '/opt/wrolpi/', '/media/wrolpi/') + WHERE destination LIKE '/opt/wrolpi/%' + """)) + if result.rowcount: + print(f"Fixed {result.rowcount} download destination paths") + + # Save corrected paths to config files directly (bypass config framework which needs SharedContext) + try: + from wrolpi.common import get_media_directory + media_dir = str(get_media_directory()) + config_dir = get_media_directory() / 'config' + + # Save domains config + domains_result = connection.execute(sa.text(""" + SELECT id, name, kind, description, directory, file_format + FROM collection + WHERE kind = 'domain' + ORDER BY name + """)) + domains_collections = [] + for row in domains_result: + collection_config = {'name': row[1], 'kind': row[2]} + if row[3]: # description + collection_config['description'] = row[3] + if row[4]: # directory + collection_config['directory'] = get_relative_path(row[4], media_dir + '/') + if row[5]: # file_format + collection_config['file_format'] = row[5] + # Get tag name if exists + tag_result = connection.execute(sa.text(""" + SELECT t.name FROM tag t + JOIN collection c ON c.tag_id = t.id + WHERE c.id = :id + """), {'id': row[0]}).first() + collection_config['tag_name'] = tag_result[0] if tag_result else None + domains_collections.append(collection_config) + + # Read existing config to preserve version + domains_file = config_dir / 'domains.yaml' + version = 0 + if domains_file.exists(): + with domains_file.open() as f: + existing = yaml.safe_load(f) or {} + version = existing.get('version', 0) + + domains_config = {'version': version + 1, 'collections': domains_collections} + save_config_file(domains_config, domains_file) + print(f"Saved domains config with {len(domains_collections)} collections") + + # Save channels config + # Columns: 0=id, 1=name, 2=kind, 3=description, 4=directory, 5=file_format, + # 6=url, 7=source_id, 8=generate_posters, 9=calculate_duration, 10=download_missing_data + channels_result = connection.execute(sa.text(""" + SELECT c.id, c.name, c.kind, c.description, c.directory, c.file_format, + ch.url, ch.source_id, ch.generate_posters, ch.calculate_duration, + ch.download_missing_data + FROM collection c + JOIN channel ch ON ch.collection_id = c.id + WHERE c.kind = 'channel' + ORDER BY c.name + """)) + channels_list = [] + for row in channels_result: + channel_config = {'name': row[1]} + if row[4]: # directory + channel_config['directory'] = get_relative_path(row[4], media_dir + '/') + if row[6]: # url + channel_config['url'] = row[6] + if row[7]: # source_id + channel_config['source_id'] = row[7] + channel_config['generate_posters'] = row[8] + channel_config['calculate_duration'] = row[9] + channel_config['download_missing_data'] = row[10] + if row[5]: # file_format + channel_config['file_format'] = row[5] + # Get tag name if exists + tag_result = connection.execute(sa.text(""" + SELECT t.name FROM tag t + JOIN collection c ON c.tag_id = t.id + WHERE c.id = :id + """), {'id': row[0]}).first() + channel_config['tag_name'] = tag_result[0] if tag_result else None + # Get downloads for this channel's collection + downloads_result = connection.execute(sa.text(""" + SELECT url, frequency FROM download + WHERE collection_id = :collection_id + """), {'collection_id': row[0]}) + downloads = [{'url': d[0], 'frequency': d[1]} for d in downloads_result] + if downloads: + channel_config['downloads'] = downloads + channels_list.append(channel_config) + + # Read existing config to preserve version + channels_file = config_dir / 'channels.yaml' + version = 0 + if channels_file.exists(): + with channels_file.open() as f: + existing = yaml.safe_load(f) or {} + version = existing.get('version', 0) + + channels_config = {'version': version + 1, 'channels': channels_list} + save_config_file(channels_config, channels_file) + print(f"Saved channels config with {len(channels_list)} channels") + + except Exception as e: + print(f"Warning: Could not save configs: {e}") + + +def downgrade(): + # Data fix, no meaningful downgrade + pass diff --git a/app/src/App.js b/app/src/App.js index 0029c0d9..d5c1bf8a 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -11,6 +11,7 @@ import {InventoryRoute} from "./components/Inventory"; import {ArchiveRoute} from "./components/Archive"; import {FilesRoute} from "./components/Files"; import {QueryProvider, StatusProvider} from "./hooks/customHooks"; +import {FileWorkerStatusProvider} from "./contexts/FileWorkerStatusContext"; import {MapRoute} from "./components/Map"; import {MediaContextProvider, mediaStyles, StatusContext, ThemeContext} from "./contexts/contexts"; import {Header, ThemeProvider} from "./components/Theme"; @@ -98,12 +99,14 @@ export default function App() { useEventsInterval(); return - {/* Context and style to handle switching between mobile/computer. */} - - {/* Toasts can be on any page. */} - - - - + + {/* Context and style to handle switching between mobile/computer. */} + + {/* Toasts can be on any page. */} + + + + + } diff --git a/app/src/api.js b/app/src/api.js index 9ce68ec3..b49fc296 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -442,6 +442,14 @@ export async function postArchiveFileFormat(archive_file_format) { return await apiPost(`${ARCHIVES_API}/file_format`, body); } +export async function fetchArchiveBrowsers() { + const response = await apiGet(`${ARCHIVES_API}/browsers`); + if (response.ok) { + return await response.json(); + } + return {browsers: [], available: false}; +} + export async function fetchBrowserProfiles() { const response = await apiGet(`${API_URI}/videos/browser-profiles`); if (response.status === 200) { @@ -888,6 +896,126 @@ export async function refreshDomain(domainId) { return response; } +export async function previewCollectionReorganization(collectionId) { + const response = await apiGet(`${COLLECTIONS_API}/${collectionId}/reorganize/preview`); + if (response.ok) { + return await response.json(); + } else { + const message = await getErrorMessage(response, 'Unable to preview reorganization. See server logs.'); + toast({ + type: 'error', + title: 'Reorganization Preview Error', + description: message, + time: 5000, + }); + throw new Error(message); + } +} + +export async function executeCollectionReorganization(collectionId) { + const response = await apiPost(`${COLLECTIONS_API}/${collectionId}/reorganize`); + if (response.ok) { + const data = await response.json(); + if (data.job_id) { + toast({ + type: 'success', + title: 'Reorganization Started', + description: 'Files are being reorganized in the background', + time: 3000, + }); + } else { + toast({ + type: 'info', + title: 'No Changes Needed', + description: data.message || 'No files need reorganization', + time: 3000, + }); + } + return data; + } else { + const message = await getErrorMessage(response, 'Unable to start reorganization. See server logs.'); + toast({ + type: 'error', + title: 'Reorganization Error', + description: message, + time: 5000, + }); + throw new Error(message); + } +} + +export async function getReorganizationStatus(collectionId, jobId) { + const response = await apiGet(`${COLLECTIONS_API}/${collectionId}/reorganize/status/${jobId}`); + if (response.ok) { + return await response.json(); + } else { + const message = await getErrorMessage(response, 'Unable to get reorganization status.'); + throw new Error(message); + } +} + +// Batch Reorganization API Functions + +export async function previewBatchReorganization(kind) { + const endpoint = kind === 'channel' ? 'channels' : 'domains'; + const response = await apiGet(`${COLLECTIONS_API}/reorganize/${endpoint}`); + if (response.ok) { + return await response.json(); + } else { + const message = await getErrorMessage(response, `Unable to get ${kind}s needing reorganization.`); + toast({ + type: 'error', + title: 'Batch Reorganization Preview Error', + description: message, + time: 5000, + }); + throw new Error(message); + } +} + +export async function executeBatchReorganization(kind) { + const endpoint = kind === 'channel' ? 'channels' : 'domains'; + const response = await apiPost(`${COLLECTIONS_API}/reorganize/${endpoint}`); + if (response.ok) { + const data = await response.json(); + if (data.batch_job_id) { + toast({ + type: 'success', + title: 'Batch Reorganization Started', + description: `${data.collection_count} ${kind}s are being reorganized`, + time: 3000, + }); + } else { + toast({ + type: 'info', + title: 'No Changes Needed', + description: data.message || `No ${kind}s need reorganization`, + time: 3000, + }); + } + return data; + } else { + const message = await getErrorMessage(response, `Unable to start batch reorganization. See server logs.`); + toast({ + type: 'error', + title: 'Batch Reorganization Error', + description: message, + time: 5000, + }); + throw new Error(message); + } +} + +export async function getBatchReorganizationStatus(batchJobId) { + const response = await apiGet(`${COLLECTIONS_API}/reorganize/batch/status/${batchJobId}`); + if (response.ok) { + return await response.json(); + } else { + const message = await getErrorMessage(response, 'Unable to get batch reorganization status.'); + throw new Error(message); + } +} + export async function getArchive(archiveId) { const response = await apiGet(`${ARCHIVES_API}/${archiveId}`); if (response.ok) { diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index f7bbf8a3..8c9486da 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -44,11 +44,13 @@ import { import { deleteArchives, deleteDomain, + fetchArchiveBrowsers, fetchArchiveDownloaderConfig, generateArchiveScreenshot, getCollectionTagInfo, postArchiveFileFormat, postDownload, + previewBatchReorganization, refreshDomain, tagDomain, tagFileGroup, @@ -57,10 +59,13 @@ import { } from "../api"; import {InputForm, useForm} from "../hooks/useForm"; import {CollectionTagModal} from "./collections/CollectionTagModal"; +import {CollectionReorganizeModal} from "./collections/CollectionReorganizeModal"; +import {BatchReorganizeModal} from "./collections/BatchReorganizeModal"; import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; import { useArchive, + useDockerized, useDomain, useDomains, useOneQuery, @@ -505,6 +510,8 @@ export function DomainEditPage() { // Modal state for tagging const [tagEditModalOpen, setTagEditModalOpen] = useState(false); + // Modal state for reorganization + const [reorganizeModalOpen, setReorganizeModalOpen] = useState(false); useTitle(`Edit Domain: ${domain?.domain || '...'}`); @@ -612,10 +619,21 @@ export function DomainEditPage() { style={{marginTop: '1em'}} >Tag; + const reorganizeButton = ( + setReorganizeModalOpen(true)} + obeyWROLMode={true} + style={{marginTop: '1em'}} + >Reorganize Files + ); + const actionButtons = <> {deleteButton} {refreshButton} {tagButton} + {reorganizeButton} ; const [descriptionProps] = form.getCustomProps({name: 'description', path: 'description'}); @@ -626,6 +644,16 @@ export function DomainEditPage() { + {domain?.needs_reorganization && ( + + File Format Changed +

+ The file name format has changed. Click "Reorganize Files" to move existing files + to match the new format. +

+
+ )} + + {/* Reorganize Modal */} + setReorganizeModalOpen(false)} + collectionId={domain?.id} + collectionName={domain?.domain} + onComplete={fetchDomain} + needsReorganization={domain?.needs_reorganization} + /> + {/* Downloads Segment */}
Downloads
@@ -726,13 +764,145 @@ function ArchiveFileNameForm({form}) { /> } +function BrowserConfigForm({form, browsers, browsersAvailable}) { + const {t} = React.useContext(ThemeContext); + const [useCustomPath, setUseCustomPath] = useState(false); + + // Determine if custom path is being used based on current value + React.useEffect(() => { + const currentBrowser = form.formData?.browser_executable; + if (currentBrowser) { + // Check if current value matches any known browser + const isKnownBrowser = browsers.some(b => b.path === currentBrowser || b.key === currentBrowser); + setUseCustomPath(!isKnownBrowser); + } + }, [form.formData?.browser_executable, browsers]); + + if (!browsersAvailable) { + return null; + } + + // Build dropdown options: auto-detect + installed browsers + custom + const browserOptions = [ + {key: 'auto', value: '', text: 'Auto-detect (recommended)'}, + ...browsers.map(b => ({key: b.key, value: b.path, text: `${b.name} (${b.path})`})), + {key: 'custom', value: '__custom__', text: 'Custom path...'}, + ]; + + const handleBrowserChange = (e, {value}) => { + if (value === '__custom__') { + setUseCustomPath(true); + form.setValue('browser_executable', ''); + } else { + setUseCustomPath(false); + form.setValue('browser_executable', value || null); + } + }; + + const currentValue = form.formData?.browser_executable || ''; + const dropdownValue = useCustomPath ? '__custom__' : currentValue; + + return <> +
Browser Settings
+

+ Configure which browser SingleFile uses to create archives. + These settings only apply to native deployments (Raspberry Pi/Debian). +

+ + + + + + + {useCustomPath && ( + + + form.setValue('browser_executable', e.target.value)} + /> + Enter the absolute path to the browser executable. + + )} + + + + form.setValue('browser_args', e.target.value)} + /> + JSON array of arguments passed to the browser. Default: ["--no-sandbox"] + + + + + form.setValue('user_agent', e.target.value || null)} + /> + Custom user agent string. Leave empty to use the system default. + + ; +} + function ArchiveSettingsPage() { useTitle('Archive Settings'); const {t} = React.useContext(ThemeContext); + const dockerized = useDockerized(); + const [batchModalOpen, setBatchModalOpen] = useState(false); + const [domainsNeedingReorg, setDomainsNeedingReorg] = useState(0); + const [fetchingReorgCount, setFetchingReorgCount] = useState(true); + const [browsers, setBrowsers] = useState([]); + const [browsersAvailable, setBrowsersAvailable] = useState(false); + + // Check how many domains need reorganization on mount + React.useEffect(() => { + setFetchingReorgCount(true); + previewBatchReorganization('domain') + .then(data => { + setDomainsNeedingReorg(data.total_collections || 0); + }) + .catch(() => { + setDomainsNeedingReorg(0); + }) + .finally(() => { + setFetchingReorgCount(false); + }); + }, []); + + // Fetch available browsers on mount (only if not dockerized) + React.useEffect(() => { + if (dockerized === false) { + fetchArchiveBrowsers() + .then(data => { + setBrowsers(data.browsers || []); + setBrowsersAvailable(data.available === true); + }) + .catch(() => { + setBrowsers([]); + setBrowsersAvailable(false); + }); + } + }, [dockerized]); const emptyFormData = { file_name_format: '%(download_datetime)s_%(title)s.%(ext)s', + browser_executable: null, + browser_args: '["--no-sandbox"]', + user_agent: null, }; const configSubmitter = async () => { @@ -769,6 +939,20 @@ function ArchiveSettingsPage() { + + {/* Browser settings - only shown on native deployments */} + {!dockerized && browsersAvailable && ( + + + + + + )} +
+ +
File Organization
+

+ {fetchingReorgCount + ? 'Checking for domains that need reorganization...' + : <> + {domainsNeedingReorg} domain{domainsNeedingReorg !== 1 ? 's' : ''} + {domainsNeedingReorg > 0 + ? ' have files that do not match the current file name format.' + : '. All domains are organized correctly.'} + + } +

+ +
+ + setBatchModalOpen(false)} + kind='domain' + onComplete={() => { + setBatchModalOpen(false); + setDomainsNeedingReorg(0); + }} + /> +
SingleFile Browser Extension
diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index fc9eed40..aa67a33a 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -2,6 +2,7 @@ import React, {useState} from "react"; import {Grid, StatisticLabel, StatisticValue,} from "semantic-ui-react"; import {createChannel, deleteChannel, refreshChannel, tagChannel, tagChannelInfo} from "../api"; import {CollectionTagModal} from "./collections/CollectionTagModal"; +import {CollectionReorganizeModal} from "./collections/CollectionReorganizeModal"; import { APIButton, BackButton, @@ -88,6 +89,7 @@ export function ChannelEditPage() { const [tagEditModalOpen, setTagEditModalOpen] = useState(false); const [downloadModalOpen, setDownloadModalOpen] = useState(false); + const [reorganizeModalOpen, setReorganizeModalOpen] = useState(false); const {channel, form, fetchChannel} = useChannel(channelId); @@ -192,10 +194,21 @@ export function ChannelEditPage() { style={{marginTop: '1em'}} >Tag; + const reorganizeButton = ( + setReorganizeModalOpen(true)} + obeyWROLMode={true} + style={{marginTop: '1em'}} + >Reorganize Files + ); + const actionButtons = <> {deleteButton} {refreshButton} {tagButton} + {reorganizeButton} ; const downloadMissingDataInfo = 'Automatically download missing comments, etc, in the background.'; @@ -207,6 +220,16 @@ export function ChannelEditPage() { + {channel?.needs_reorganization && ( + + File Format Changed +

+ The file name format has changed. Click "Reorganize Files" to move existing files + to match the new format. +

+
+ )} + + {/* Reorganize Modal */} + setReorganizeModalOpen(false)} + collectionId={channel?.collection_id} + collectionName={channel?.name} + onComplete={fetchChannel} + needsReorganization={channel?.needs_reorganization} + /> + {/* Downloads Segment */} diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 6fb0f27e..4eeb148e 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -780,7 +780,7 @@ export function InfoPopup({ position = 'left center', iconSize = null, header = '', - iconStyle = {marginLeft: '0.25em', marginRight: '0.25em', marginTop: '-0.5em'}, + iconStyle = { margin: '0.5em'}, ...props }) { const trigger = ; diff --git a/app/src/components/DomainEditPage.test.js b/app/src/components/DomainEditPage.test.js index 2038a3ef..c767d78b 100644 --- a/app/src/components/DomainEditPage.test.js +++ b/app/src/components/DomainEditPage.test.js @@ -54,6 +54,19 @@ jest.mock('./collections/CollectionTagModal', () => ({ }, })); +// Mock CollectionReorganizeModal +jest.mock('./collections/CollectionReorganizeModal', () => ({ + CollectionReorganizeModal: ({open, onClose, collectionId, collectionName, onComplete}) => { + if (!open) return null; + return ( +
+
Reorganize Files: {collectionName}
+ +
+ ); + }, +})); + // Mock API functions const mockGetCollectionTagInfo = jest.fn(); jest.mock('../api', () => ({ diff --git a/app/src/components/Files.js b/app/src/components/Files.js index aa5c10a6..492d3686 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -33,7 +33,6 @@ import { useTitle } from "./Common"; import { - useFilesProgressInterval, usePages, useSearchFiles, useSearchFilter, @@ -41,6 +40,7 @@ import { useStatusFlag, useWROLMode } from "../hooks/customHooks"; +import {useFileWorkerStatus} from "../contexts/FileWorkerStatusContext"; import {Route, Routes} from "react-router"; import {CardPlaceholder} from "./Placeholder"; import {ArchiveCard, ArchiveRowCells} from "./Archive"; @@ -566,7 +566,7 @@ function getPhaseLabel(status) { } export function FilesRefreshProgress() { - const {progress} = useFilesProgressInterval(); + const {status: progress} = useFileWorkerStatus(); if (!progress || progress.status === 'idle') { return null; diff --git a/app/src/components/Nav.js b/app/src/components/Nav.js index c815f7a8..7f9c8374 100644 --- a/app/src/components/Nav.js +++ b/app/src/components/Nav.js @@ -5,6 +5,7 @@ import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/c import {DarkModeToggle, HotspotStatusIcon, useLocalStorage} from "./Common"; import {ShareButton} from "./Share"; import {useCPUTemperature, useIOStats, useLoad, useMemoryStats, usePowerStats, useWROLMode} from "../hooks/customHooks"; +import {useReorganizationStatus} from "../contexts/FileWorkerStatusContext"; import {SearchIconButton} from "./Search"; import {Icon, Popup} from "./Theme"; import {HELP_VIEWER_URI, NAME} from "./Vars"; @@ -165,10 +166,33 @@ export function NavBar() { // issue. const warningIcon = temperatureIcon || diskWaitIcon || memoryIcon || systemLoadIcon; + const {isReorganizing, taskType, collectionId, collectionKind} = useReorganizationStatus(); + let processingLink; if (status && status.flags) { if (status.flags.file_worker_busy) { - processingLink = '/files'; + // Smart navigation based on what the file worker is doing + if (isReorganizing && collectionId && collectionKind) { + // Single collection reorganization - navigate to that collection + if (collectionKind === 'channel') { + processingLink = `/videos/channel/${collectionId}/edit`; + } else if (collectionKind === 'domain') { + processingLink = `/archive/domain/${collectionId}/edit`; + } else { + processingLink = '/files'; + } + } else if (isReorganizing && taskType === 'batch_reorganize' && collectionKind) { + // Batch reorganization - navigate to settings page for that kind + if (collectionKind === 'channel') { + processingLink = '/videos/settings'; + } else if (collectionKind === 'domain') { + processingLink = '/archive/settings'; + } else { + processingLink = '/files'; + } + } else { + processingLink = '/files'; + } } else if (status.flags.map_importing) { processingLink = '/map/manage'; } diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 02f31e14..60e3c2bf 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -57,6 +57,8 @@ import _ from "lodash"; import {defaultFileOrder, defaultSearchOrder, HELP_VIEWER_URI} from "./Vars"; import {InputForm, ToggleForm, useForm} from "../hooks/useForm"; import {VideoResolutionSelectorForm} from "./Download"; +import {BatchReorganizeModal} from "./collections/BatchReorganizeModal"; +import {previewBatchReorganization} from "../api"; export function VideoWrapper() { const {videoId} = useParams(); @@ -234,15 +236,25 @@ function VideosPage() { function VideoFileNameForm({form}) { const [message, setMessage] = React.useState(null); - const onChange = async (value) => { - const response = await postVideoFileFormat(value); - const {error, preview} = await response.json(); - if (error) { - setMessage({content: error, header: 'Invalid File Name', negative: true}); - } else { - setMessage({content: preview, header: 'File Name Preview', positive: true}); - } - } + const debouncedOnChange = React.useCallback( + _.debounce(async (value) => { + const response = await postVideoFileFormat(value); + const {error, preview} = await response.json(); + if (error) { + setMessage({content: error, header: 'Invalid File Name', negative: true}); + } else { + setMessage({content: preview, header: 'File Name Preview', positive: true}); + } + }, 300), + [] + ); + + // Cleanup debounce on unmount + React.useEffect(() => { + return () => debouncedOnChange.cancel(); + }, [debouncedOnChange]); + + const onChange = debouncedOnChange; const label = %(upload_year)s - Upload year (YYYY)
  • %(upload_month)s - Upload month (01-12)
  • %(id)s - Video ID
  • -
  • %(ext)s - File extension
  • %(channel)s - Channel name
  • %(duration)s - Duration in seconds
  • %(playlist_index)s - Playlist index
  • +
  • %(ext)s - File extension (required, must be at end)
  • See yt-dlp docs for all variables. Subdirectories supported.

    } @@ -279,6 +291,25 @@ function VideoFileNameForm({form}) { function VideosSettingsPage() { useTitle('Videos Settings'); + const [batchModalOpen, setBatchModalOpen] = useState(false); + const [channelsNeedingReorg, setChannelsNeedingReorg] = useState(0); + const [fetchingReorgCount, setFetchingReorgCount] = useState(true); + + // Check how many channels need reorganization on mount + useEffect(() => { + setFetchingReorgCount(true); + previewBatchReorganization('channel') + .then(data => { + setChannelsNeedingReorg(data.total_collections || 0); + }) + .catch(() => { + setChannelsNeedingReorg(0); + }) + .finally(() => { + setFetchingReorgCount(false); + }); + }, []); + const emptyFormData = { video_resolutions: ['1080p', '720p', '480p', 'maximum'], yt_dlp_options: { @@ -337,10 +368,11 @@ function VideosSettingsPage() { popupContent='Always download videos with the selected browser profile. This is risky, and therefore discouraged.' />; - return -
    Video Downloader Config
    + return <> + +
    Video Downloader Config
    -
    + @@ -435,7 +467,42 @@ function VideosSettingsPage() {
    -
    +
    + + +
    File Organization
    +

    + {fetchingReorgCount + ? 'Checking for channels that need reorganization...' + : <> + {channelsNeedingReorg} channel{channelsNeedingReorg !== 1 ? 's' : ''} + {channelsNeedingReorg > 0 + ? ' have files that do not match the current file name format.' + : '. All channels are organized correctly.'} + + } +

    + +
    + + setBatchModalOpen(false)} + kind='channel' + onComplete={() => { + setBatchModalOpen(false); + setChannelsNeedingReorg(0); + }} + /> + } diff --git a/app/src/components/admin/Status.js b/app/src/components/admin/Status.js index 7733b12c..e70d96ad 100644 --- a/app/src/components/admin/Status.js +++ b/app/src/components/admin/Status.js @@ -303,15 +303,15 @@ export function StatusPage() { {cpuProgress} {memoryUsageProgress} + + + - - - @@ -322,6 +322,9 @@ export function StatusPage() { {cpuProgress} {memoryUsageProgress} + + + - - - diff --git a/app/src/components/collections/BatchReorganizeModal.js b/app/src/components/collections/BatchReorganizeModal.js new file mode 100644 index 00000000..c6455736 --- /dev/null +++ b/app/src/components/collections/BatchReorganizeModal.js @@ -0,0 +1,411 @@ +import React, {useCallback, useEffect, useState} from 'react'; +import {Grid, Icon, Label, Progress} from 'semantic-ui-react'; +import Message from 'semantic-ui-react/dist/commonjs/collections/Message'; +import {Button, Modal, Table} from '../Theme'; +import {APIButton} from '../Common'; +import { + executeBatchReorganization, + getBatchReorganizationStatus, + previewBatchReorganization +} from '../../api'; +import {useReorganizationStatus, useFileWorkerStatus} from '../../contexts/FileWorkerStatusContext'; + +/** + * Modal component for previewing and executing batch collection file reorganization. + * + * Shows a list of collections that need reorganization, allows user to execute + * batch reorganization, and tracks progress with dual progress bars (overall and per-collection). + * + * @param {boolean} open - Whether the modal is open + * @param {Function} onClose - Callback when modal is closed + * @param {string} kind - 'channel' or 'domain' + * @param {Function} onComplete - Optional callback when reorganization completes + */ +export function BatchReorganizeModal({ + open, + onClose, + kind, + onComplete, + }) { + const [loading, setLoading] = useState(false); + const [preview, setPreview] = useState(null); + const [error, setError] = useState(null); + const [batchJobId, setBatchJobId] = useState(null); + const [status, setStatus] = useState(null); + const [polling, setPolling] = useState(false); + + const kindLabel = kind === 'channel' ? 'Channel' : 'Domain'; + const kindLabelPlural = kind === 'channel' ? 'Channels' : 'Domains'; + + // Check if batch reorganization is currently active + const { + isReorganizing, + taskType, + collectionKind, + batchStatus: workerBatchStatus, + workerStatus, + } = useReorganizationStatus(); + const {setFastPolling} = useFileWorkerStatus(); + + // Enable fast polling when modal is open + useEffect(() => { + if (open) { + setFastPolling(true); + } + return () => setFastPolling(false); + }, [open, setFastPolling]); + + // Load preview when modal opens, or resume if already reorganizing + useEffect(() => { + if (open && kind) { + // Check if batch_status has an active job for this kind + // This is the primary indicator since worker status can briefly show + // 'reorganizing'/'reorganize' during individual collection processing + const hasActiveBatchJob = workerBatchStatus?.batch_job_id && + collectionKind === kind; + + // Worker is actively doing something (taskType is set) + // After batch completes, taskType becomes null, so this distinguishes + // "batch in progress" from "batch completed but batch_job_id still cached" + const workerIsActive = taskType != null; + + // Use either the standard detection OR the batch_status fallback (only when worker is active) + const isThisActive = (isReorganizing && taskType === 'batch_reorganize' && collectionKind === kind) || + (hasActiveBatchJob && workerIsActive); + + if (isThisActive && workerBatchStatus) { + // Resume: start polling API immediately for accurate progress + // Don't use workerBatchStatus.current_collection directly as it may have stale + // percent/total/completed values. The batch status API returns fresh values + // by copying operation_percent into current_collection.percent. + setLoading(false); + setPreview(null); + setBatchJobId(workerBatchStatus.batch_job_id || null); + // Don't setStatus from workerBatchStatus - let the first poll populate it + // with accurate values from the batch status API + setPolling(true); + } else if (!polling && !batchJobId) { + // Normal: fetch preview (only if not already polling/resuming) + setLoading(true); + setError(null); + setPreview(null); + setBatchJobId(null); + setStatus(null); + + previewBatchReorganization(kind) + .then(data => { + setPreview(data); + setLoading(false); + }) + .catch(err => { + setError(err.message); + setLoading(false); + }); + } + } + }, [open, kind, isReorganizing, taskType, collectionKind]); + + // Poll for status when we have a batch job ID + useEffect(() => { + if (!batchJobId || !polling) { + return; + } + + let timeoutId = null; + let cancelled = false; + + const pollStatus = async () => { + if (cancelled) { + return; + } + + try { + const statusData = await getBatchReorganizationStatus(batchJobId); + if (cancelled) return; + + setStatus(statusData); + + if (statusData.status === 'complete') { + setPolling(false); + if (onComplete) { + onComplete(); + } + } else if (statusData.status === 'failed' || statusData.error) { + setPolling(false); + setError(statusData.error || 'Batch reorganization failed'); + } else { + // Schedule next poll after this one completes + timeoutId = setTimeout(pollStatus, 500); + } + } catch (err) { + console.error('Failed to get batch reorganization status', err); + if (!cancelled) { + // Retry after delay even on error + timeoutId = setTimeout(pollStatus, 1000); + } + } + }; + + // Start polling immediately + pollStatus(); + + return () => { + cancelled = true; + if (timeoutId) { + clearTimeout(timeoutId); + } + }; + }, [batchJobId, polling, onComplete]); + + const handleClose = useCallback(() => { + // Allow closing - user can reopen to resume if needed + setPreview(null); + setError(null); + setBatchJobId(null); + setStatus(null); + setPolling(false); + onClose(); + }, [onClose]); + + const handleReorganize = useCallback(async () => { + try { + setError(null); + const result = await executeBatchReorganization(kind); + if (result.batch_job_id) { + setBatchJobId(result.batch_job_id); + setPolling(true); + setStatus({ + status: 'pending', + total_collections: result.collection_count, + completed_collections: 0, + current_collection: null, + overall_percent: 0, + completed: [], + failed_collection: null, + error: null, + }); + } else { + // No collections needed reorganization + if (onComplete) { + onComplete(); + } + handleClose(); + } + } catch (err) { + setError(err.message); + } + }, [kind, onComplete, handleClose]); + + const renderPreview = () => { + if (!preview) return null; + + return ( + + + +

    + {preview.total_collections} {kindLabelPlural.toLowerCase()} need reorganization, + with {preview.total_files_needing_move} total files to move. +

    +

    + New format: +

    +
    +
    + + {preview.collections && preview.collections.length > 0 && ( + + + {kindLabelPlural} to reorganize: + + + + {kindLabel} + Files to Move + Sample + + + + {preview.collections.slice(0, 10).map((collection, idx) => ( + + {collection.collection_name} + + {collection.files_needing_move} / {collection.total_files} + + + {collection.sample_move ? ( + <> + {collection.sample_move.old_path} +
    + + {collection.sample_move.new_path} + + ) : '-'} +
    +
    + ))} +
    +
    + {preview.collections.length > 10 && ( +

    + ...and {preview.collections.length - 10} more {kindLabelPlural.toLowerCase()} +

    + )} +
    +
    + )} + + {preview.total_collections === 0 && ( + + + + No {kindLabelPlural} Need Reorganization +

    All {kindLabelPlural.toLowerCase()} are already organized according to the current format.

    +
    +
    +
    + )} +
    + ); + }; + + const renderProgress = () => { + if (!status) return null; + + const currentCollection = status.current_collection; + + return ( + + + +

    + Overall Status: {status.status} +

    + Overall Progress: + +

    + {status.completed_collections || 0} of {status.total_collections || 0} {kindLabelPlural.toLowerCase()} completed +

    +
    +
    + + {currentCollection && ( + + + Currently Processing: {currentCollection.name} + +

    + {currentCollection.completed || 0} of {currentCollection.total || 0} files +

    +
    +
    + )} + + {status.completed && status.completed.length > 0 && ( + + + Completed: +
    + {status.completed.map((c, idx) => ( + + ))} +
    +
    +
    + )} + + {status.status === 'complete' && ( + + + + Batch Reorganization Complete +

    All {kindLabelPlural.toLowerCase()} have been reorganized successfully.

    +
    +
    +
    + )} + + {status.status === 'failed' && status.failed_collection && ( + + + + Reorganization Failed +

    + Failed on {kindLabel.toLowerCase()}: {status.failed_collection.name} +

    + {status.error &&

    Error: {status.error}

    } +
    +
    +
    + )} +
    + ); + }; + + const isInProgress = polling || (status && status.status !== 'complete' && status.status !== 'failed'); + const canReorganize = preview && preview.total_collections > 0 && !isInProgress && !batchJobId; + + return ( + + + Reorganize All {kindLabelPlural} + + + {loading && ( + + + + Loading Preview + Analyzing {kindLabelPlural.toLowerCase()}... + + + )} + + {error && !status?.failed_collection && ( + + Error +

    {error}

    +
    + )} + + {!loading && !batchJobId && renderPreview()} + {batchJobId && renderProgress()} +
    + + + {canReorganize && ( + + Reorganize All {kindLabelPlural} + + )} + +
    + ); +} diff --git a/app/src/components/collections/BatchReorganizeModal.test.js b/app/src/components/collections/BatchReorganizeModal.test.js new file mode 100644 index 00000000..53904828 --- /dev/null +++ b/app/src/components/collections/BatchReorganizeModal.test.js @@ -0,0 +1,316 @@ +import React from 'react'; +import {screen, waitFor, fireEvent} from '@testing-library/react'; +import {renderWithProviders} from '../../test-utils'; +import {BatchReorganizeModal} from './BatchReorganizeModal'; +import * as api from '../../api'; + +// Mock the API functions +jest.mock('../../api', () => ({ + previewBatchReorganization: jest.fn(), + executeBatchReorganization: jest.fn(), + getBatchReorganizationStatus: jest.fn(), +})); + +// Mock react-semantic-toasts-2 +jest.mock('react-semantic-toasts-2', () => ({ + toast: jest.fn(), +})); + +// Mock the FileWorkerStatusContext +jest.mock('../../contexts/FileWorkerStatusContext', () => ({ + useReorganizationStatus: () => ({ + isReorganizing: false, + taskType: null, + collectionId: null, + collectionKind: null, + batchStatus: null, + workerStatus: null, + refresh: jest.fn(), + }), + useFileWorkerStatus: () => ({ + status: null, + error: null, + refresh: jest.fn(), + setFastPolling: jest.fn(), + }), +})); + +describe('BatchReorganizeModal', () => { + const defaultProps = { + open: true, + onClose: jest.fn(), + kind: 'channel', + onComplete: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('loads preview when opened', async () => { + const mockPreview = { + collections: [ + { + collection_id: 1, + collection_name: 'Test Channel', + total_files: 50, + files_needing_move: 45, + sample_move: {old_path: 'old/path.mp4', new_path: 'new/path.mp4'}, + }, + ], + total_collections: 1, + total_files_needing_move: 45, + new_file_format: '%(upload_date)s/%(title)s.%(ext)s', + }; + + api.previewBatchReorganization.mockResolvedValue(mockPreview); + + renderWithProviders(); + + // Should show loading initially + expect(screen.getByText(/Loading Preview/i)).toBeInTheDocument(); + + // Wait for preview to load + await waitFor(() => { + expect(screen.getByText(/Test Channel/)).toBeInTheDocument(); + }); + + expect(api.previewBatchReorganization).toHaveBeenCalledWith('channel'); + }); + + it('shows collection list with sample moves', async () => { + const mockPreview = { + collections: [ + { + collection_id: 1, + collection_name: 'Channel A', + total_files: 100, + files_needing_move: 80, + sample_move: {old_path: 'videos/old.mp4', new_path: 'videos/new.mp4'}, + }, + { + collection_id: 2, + collection_name: 'Channel B', + total_files: 50, + files_needing_move: 30, + sample_move: {old_path: 'videos/old2.mp4', new_path: 'videos/new2.mp4'}, + }, + ], + total_collections: 2, + total_files_needing_move: 110, + new_file_format: '%(upload_date)s/%(title)s.%(ext)s', + }; + + api.previewBatchReorganization.mockResolvedValue(mockPreview); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText('Channel A')).toBeInTheDocument(); + expect(screen.getByText('Channel B')).toBeInTheDocument(); + }); + + // Should show total statistics - look for the numbers in the text context + expect(screen.getByText('2')).toBeInTheDocument(); // total collections + expect(screen.getByText('110')).toBeInTheDocument(); // total files + }); + + it('shows total statistics', async () => { + const mockPreview = { + collections: [ + {collection_id: 1, collection_name: 'ChannelA', total_files: 50, files_needing_move: 45, sample_move: null}, + {collection_id: 2, collection_name: 'ChannelB', total_files: 30, files_needing_move: 25, sample_move: null}, + {collection_id: 3, collection_name: 'ChannelC', total_files: 20, files_needing_move: 15, sample_move: null}, + ], + total_collections: 3, + total_files_needing_move: 85, + new_file_format: '%(title)s.%(ext)s', + }; + + api.previewBatchReorganization.mockResolvedValue(mockPreview); + + renderWithProviders(); + + await waitFor(() => { + // Look for channel names to verify list is displayed + expect(screen.getByText('ChannelA')).toBeInTheDocument(); + expect(screen.getByText('ChannelB')).toBeInTheDocument(); + expect(screen.getByText('ChannelC')).toBeInTheDocument(); + }); + }); + + it('executes batch reorganization on button click', async () => { + const mockPreview = { + collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}], + total_collections: 1, + total_files_needing_move: 5, + new_file_format: '%(title)s.%(ext)s', + }; + + api.previewBatchReorganization.mockResolvedValue(mockPreview); + api.executeBatchReorganization.mockResolvedValue({ + batch_job_id: 'batch-reorganize-abc123', + message: 'Batch reorganization started', + collection_count: 1, + }); + api.getBatchReorganizationStatus.mockResolvedValue({ + status: 'complete', + total_collections: 1, + completed_collections: 1, + current_collection: null, + overall_percent: 100, + completed: [{id: 1, name: 'Test', status: 'complete'}], + failed_collection: null, + error: null, + }); + + renderWithProviders(); + + // Wait for preview to load + await waitFor(() => { + expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument(); + }); + + // Click the reorganize button + fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i})); + + await waitFor(() => { + expect(api.executeBatchReorganization).toHaveBeenCalledWith('channel'); + }); + }); + + it('shows overall progress bar during execution', async () => { + const mockPreview = { + collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}], + total_collections: 1, + total_files_needing_move: 5, + new_file_format: '%(title)s.%(ext)s', + }; + + api.previewBatchReorganization.mockResolvedValue(mockPreview); + api.executeBatchReorganization.mockResolvedValue({ + batch_job_id: 'batch-reorganize-abc123', + message: 'Batch reorganization started', + collection_count: 1, + }); + api.getBatchReorganizationStatus.mockResolvedValue({ + status: 'running', + total_collections: 1, + completed_collections: 0, + current_collection: {id: 1, name: 'Test', status: 'running', total: 5, completed: 2, percent: 40}, + overall_percent: 40, + completed: [], + failed_collection: null, + error: null, + }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument(); + }); + + fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i})); + + await waitFor(() => { + expect(screen.getByText(/Overall Progress/i)).toBeInTheDocument(); + }); + }); + + it('shows per-collection progress bars', async () => { + const mockPreview = { + collections: [{collection_id: 1, collection_name: 'Test Channel', total_files: 10, files_needing_move: 5, sample_move: null}], + total_collections: 1, + total_files_needing_move: 5, + new_file_format: '%(title)s.%(ext)s', + }; + + api.previewBatchReorganization.mockResolvedValue(mockPreview); + api.executeBatchReorganization.mockResolvedValue({ + batch_job_id: 'batch-reorganize-abc123', + message: 'Batch reorganization started', + collection_count: 1, + }); + api.getBatchReorganizationStatus.mockResolvedValue({ + status: 'running', + total_collections: 1, + completed_collections: 0, + current_collection: {id: 1, name: 'Test Channel', status: 'running', total: 5, completed: 2, percent: 40}, + overall_percent: 20, + completed: [], + failed_collection: null, + error: null, + }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument(); + }); + + fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i})); + + await waitFor(() => { + expect(screen.getByText(/Currently Processing/i)).toBeInTheDocument(); + expect(screen.getByText(/Test Channel/)).toBeInTheDocument(); + }); + }); + + it('calls onComplete when finished', async () => { + const mockOnComplete = jest.fn(); + const mockPreview = { + collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}], + total_collections: 1, + total_files_needing_move: 5, + new_file_format: '%(title)s.%(ext)s', + }; + + api.previewBatchReorganization.mockResolvedValue(mockPreview); + api.executeBatchReorganization.mockResolvedValue({ + batch_job_id: 'batch-reorganize-abc123', + message: 'Batch reorganization started', + collection_count: 1, + }); + api.getBatchReorganizationStatus.mockResolvedValue({ + status: 'complete', + total_collections: 1, + completed_collections: 1, + current_collection: null, + overall_percent: 100, + completed: [{id: 1, name: 'Test', status: 'complete'}], + failed_collection: null, + error: null, + }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument(); + }); + + fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i})); + + await waitFor(() => { + expect(mockOnComplete).toHaveBeenCalled(); + }); + }); + + it('handles domain kind correctly', async () => { + const mockPreview = { + collections: [{collection_id: 1, collection_name: 'example.com', total_files: 10, files_needing_move: 5, sample_move: null}], + total_collections: 1, + total_files_needing_move: 5, + new_file_format: '%(title)s.%(ext)s', + }; + + api.previewBatchReorganization.mockResolvedValue(mockPreview); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByText(/Reorganize All Domains/i)).toBeInTheDocument(); + }); + + expect(api.previewBatchReorganization).toHaveBeenCalledWith('domain'); + }); +}); diff --git a/app/src/components/collections/CollectionReorganizeModal.js b/app/src/components/collections/CollectionReorganizeModal.js new file mode 100644 index 00000000..03d11ccb --- /dev/null +++ b/app/src/components/collections/CollectionReorganizeModal.js @@ -0,0 +1,345 @@ +import React, {useCallback, useEffect, useState} from 'react'; +import {Grid, Icon, Label, Progress} from 'semantic-ui-react'; +import Message from 'semantic-ui-react/dist/commonjs/collections/Message'; +import {Button, Modal, Table} from '../Theme'; +import {APIButton} from '../Common'; +import { + executeCollectionReorganization, + getReorganizationStatus, + previewCollectionReorganization +} from '../../api'; +import {useReorganizationStatus} from '../../contexts/FileWorkerStatusContext'; + +/** + * Modal component for previewing and executing collection file reorganization. + * + * Shows a preview of files that will be moved when the file_name_format changes, + * allows user to execute the reorganization, and tracks progress. + * + * @param {boolean} open - Whether the modal is open + * @param {Function} onClose - Callback when modal is closed + * @param {number} collectionId - ID of the collection to reorganize + * @param {string} collectionName - Name of the collection for display + * @param {Function} onComplete - Optional callback when reorganization completes + * @param {boolean} needsReorganization - Whether the collection.needs_reorganization flag is true + */ +export function CollectionReorganizeModal({ + open, + onClose, + collectionId, + collectionName, + onComplete, + needsReorganization, + }) { + const [loading, setLoading] = useState(false); + const [preview, setPreview] = useState(null); + const [error, setError] = useState(null); + const [jobId, setJobId] = useState(null); + const [status, setStatus] = useState(null); + const [polling, setPolling] = useState(false); + + // Check if this collection is currently being reorganized + const { + isReorganizing, + taskType, + collectionId: activeCollectionId, + workerStatus, + } = useReorganizationStatus(); + + // Load preview when modal opens, or resume if already reorganizing + useEffect(() => { + if (open && collectionId) { + // Check if THIS collection is currently being reorganized + const isThisActive = isReorganizing && + taskType === 'reorganize' && + activeCollectionId === collectionId; + + if (isThisActive) { + // Resume: show current progress from worker status + setLoading(false); + setPreview(null); + setJobId('active'); // Marker for active job + setStatus({ + status: workerStatus.status, + total: workerStatus.operation_total, + completed: workerStatus.operation_processed, + percent: workerStatus.operation_percent, + }); + setPolling(true); + } else if (!polling && !jobId) { + // Normal: fetch preview (only if not already polling/resuming) + setLoading(true); + setError(null); + setPreview(null); + setJobId(null); + setStatus(null); + + previewCollectionReorganization(collectionId) + .then(data => { + setPreview(data); + setLoading(false); + }) + .catch(err => { + setError(err.message); + setLoading(false); + }); + } + } + }, [open, collectionId, isReorganizing, taskType, activeCollectionId]); + + // Poll for status when we have a job ID + // Uses setTimeout instead of setInterval to wait for each request to complete + // before scheduling the next one, preventing request pile-up on slow connections + useEffect(() => { + if (!jobId || !polling) return; + + let timeoutId = null; + let cancelled = false; + + const pollStatus = async () => { + if (cancelled) return; + + try { + const statusData = await getReorganizationStatus(collectionId, jobId); + if (cancelled) return; + + setStatus(statusData); + + if (statusData.status === 'complete') { + setPolling(false); + if (onComplete) { + onComplete(); + } + } else if (statusData.status === 'error' || statusData.error) { + setPolling(false); + setError(statusData.error || 'Reorganization failed'); + } else { + // Schedule next poll after this one completes + timeoutId = setTimeout(pollStatus, 1000); + } + } catch (err) { + console.error('Failed to get reorganization status', err); + if (!cancelled) { + // Retry after delay even on error + timeoutId = setTimeout(pollStatus, 1000); + } + } + }; + + // Start polling immediately + pollStatus(); + + return () => { + cancelled = true; + if (timeoutId) { + clearTimeout(timeoutId); + } + }; + }, [jobId, polling, collectionId, onComplete]); + + const handleClose = useCallback(() => { + // Allow closing - user can reopen to resume if needed + setPreview(null); + setError(null); + setJobId(null); + setStatus(null); + setPolling(false); + onClose(); + }, [onClose]); + + const handleReorganize = useCallback(async () => { + try { + setError(null); + const result = await executeCollectionReorganization(collectionId); + if (result.job_id) { + setJobId(result.job_id); + setPolling(true); + setStatus({status: 'pending', total: 0, completed: 0, percent: 0}); + } else { + // No files needed reorganization + if (onComplete) { + onComplete(); + } + handleClose(); + } + } catch (err) { + setError(err.message); + } + }, [collectionId, onComplete, handleClose]); + + const renderPreview = () => { + if (!preview) return null; + + return ( + + {/* Warning when collection appears organized but has files to move */} + {preview.files_needing_move > 0 && !needsReorganization && ( + + + + Collection Appears Organized +

    + This collection's format matches the current configuration, but + {' '}{preview.files_needing_move} files don't match the expected layout. + This can happen if a previous reorganization was interrupted. +

    +
    +
    +
    + )} + + +

    + {preview.files_needing_move} of {preview.total_files} files + will be reorganized. +

    + {preview.current_file_format && ( +

    + Current format: {preview.current_file_format} +

    + )} +

    + New format: +

    +
    +
    + + {preview.sample_moves && preview.sample_moves.length > 0 && ( + + + Sample moves: + + + + Current Path + + New Path + + + + {preview.sample_moves.map((move, idx) => ( + + + {move.old_path} + + + + + + {move.new_path} + + + ))} + +
    + {preview.files_needing_move > preview.sample_moves.length && ( +

    + ...and {preview.files_needing_move - preview.sample_moves.length} more files +

    + )} +
    +
    + )} + + {preview.files_needing_move === 0 && ( + + + + No Files Need Reorganization +

    All files are already organized according to the current format.

    +
    +
    +
    + )} +
    + ); + }; + + const renderProgress = () => { + if (!status) return null; + + return ( + + + +

    + Status: {status.status} +

    + +

    + {status.completed || 0} of {status.total || 0} files processed +

    +
    +
    + + {status.status === 'complete' && ( + + + + Reorganization Complete +

    All files have been reorganized successfully.

    +
    +
    +
    + )} +
    + ); + }; + + const isInProgress = polling || (status && status.status !== 'complete' && status.status !== 'error'); + const canReorganize = preview && preview.files_needing_move > 0 && !isInProgress && !jobId; + + return ( + + + Reorganize Files: {collectionName} + + + {loading && ( + + + + Loading Preview + Analyzing files... + + + )} + + {error && ( + + Error +

    {error}

    +
    + )} + + {!loading && !jobId && renderPreview()} + {jobId && renderProgress()} +
    + + + {canReorganize && ( + + Reorganize Files + + )} + +
    + ); +} diff --git a/app/src/contexts/FileWorkerStatusContext.js b/app/src/contexts/FileWorkerStatusContext.js new file mode 100644 index 00000000..bac4fc91 --- /dev/null +++ b/app/src/contexts/FileWorkerStatusContext.js @@ -0,0 +1,67 @@ +import React, {createContext, useContext, useState, useEffect, useCallback, useRef} from 'react'; +import {fetchFilesProgress} from '../api'; + +const FileWorkerStatusContext = createContext(null); + +export function FileWorkerStatusProvider({children, pollInterval = 3000}) { + const [status, setStatus] = useState(null); + const [error, setError] = useState(null); + const requestedInterval = useRef(pollInterval); + + const refresh = useCallback(async () => { + try { + const data = await fetchFilesProgress(); + setStatus(data); + setError(null); + } catch (err) { + setError(err); + } + }, []); + + // Allow components to request faster polling temporarily + const setFastPolling = useCallback((fast) => { + requestedInterval.current = fast ? 500 : pollInterval; + }, [pollInterval]); + + useEffect(() => { + let timeoutId; + const poll = async () => { + await refresh(); + timeoutId = setTimeout(poll, requestedInterval.current); + }; + poll(); + return () => clearTimeout(timeoutId); + }, [refresh]); + + return ( + + {children} + + ); +} + +export function useFileWorkerStatus() { + const context = useContext(FileWorkerStatusContext); + if (!context) { + throw new Error('useFileWorkerStatus must be used within FileWorkerStatusProvider'); + } + return context; +} + +// Convenience hook for reorganization-specific status +export function useReorganizationStatus() { + const {status: progress, refresh} = useFileWorkerStatus(); + + const isReorganizing = progress?.status === 'reorganizing' || + progress?.status === 'batch_reorganizing'; + + return { + isReorganizing, + taskType: progress?.task_type, + collectionId: progress?.collection_id, + collectionKind: progress?.collection_kind, + batchStatus: progress?.batch_status, + workerStatus: progress, + refresh, + }; +} diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 7f9ffb4c..c0a132af 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -5,7 +5,6 @@ import { fetchChannels, fetchDecoded, fetchDomains, - fetchFilesProgress, filesSearch, getArchive, getChannel, @@ -588,7 +587,7 @@ export const useChannel = (channel_id) => { return { channel: form.formData, form, - fetchChannel, + fetchChannel: form.fetcher, }; } @@ -760,34 +759,6 @@ export const useBrowseFiles = () => { return {browseFiles, openFolders, setOpenFolders, fetchFiles}; } -export const useFilesProgress = () => { - const [progress, setProgress] = useState(null); - - const localFetchFilesProgress = async () => { - try { - let p = await fetchFilesProgress(); - setProgress(p); - } catch (e) { - setProgress(null); - console.error(e); - } - } - - useEffect(() => { - localFetchFilesProgress(); - }, []); - - return {progress, fetchFilesProgress: localFetchFilesProgress} -} - -export const useFilesProgressInterval = () => { - const {progress, fetchFilesProgress} = useFilesProgress(); - - useRecurringTimeout(fetchFilesProgress, 1000); - - return {progress, fetchFilesProgress}; -} - export const useHotspot = () => { const [on, setOn] = useState(null); const [inUse, setInUse] = useState(false); diff --git a/app/src/test-utils.js b/app/src/test-utils.js index 27ac7342..9966ad41 100644 --- a/app/src/test-utils.js +++ b/app/src/test-utils.js @@ -3,11 +3,50 @@ * Provides custom render functions with necessary context providers */ -import React from 'react'; +import React, {createContext, useContext} from 'react'; import {render} from '@testing-library/react'; import {BrowserRouter} from 'react-router'; import {MediaContextProvider, ThemeContext} from './contexts/contexts'; +// Mock FileWorkerStatusContext for tests - avoids API calls during testing +const MockFileWorkerStatusContext = createContext(null); + +function MockFileWorkerStatusProvider({children}) { + const mockValue = { + status: null, + error: null, + refresh: jest.fn(), + setFastPolling: jest.fn(), + }; + return ( + + {children} + + ); +} + +// Re-export hooks that use the mock context +export function useFileWorkerStatus() { + const context = useContext(MockFileWorkerStatusContext); + if (!context) { + throw new Error('useFileWorkerStatus must be used within FileWorkerStatusProvider'); + } + return context; +} + +export function useReorganizationStatus() { + const {status: progress, refresh} = useFileWorkerStatus(); + return { + isReorganizing: false, + taskType: null, + collectionId: null, + collectionKind: null, + batchStatus: null, + workerStatus: progress, + refresh, + }; +} + /** * Custom render function that wraps components with necessary providers * @param {React.Component} ui - Component to render @@ -40,9 +79,11 @@ export function renderWithProviders( function Wrapper({children}) { const content = ( - - {children} - + + + {children} + + ); diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 6876b9a5..645d6c20 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -7,7 +7,7 @@ from typing import List, Tuple, Iterable from sqlalchemy import not_ from sqlalchemy.orm import Session -from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM +from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM, FIREFOX from wrolpi.collections import Collection from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \ slow_logger, get_title_from_html, TRACE_LEVEL @@ -86,11 +86,29 @@ class ArchiveDownloader(Downloader, ABC): async def do_singlefile(self, download: Download) -> bytes: """Create a Singlefile from the archive's URL.""" + # Get browser settings from config + config = lib.get_archive_downloader_config() + + # Determine browser executable + browser_executable = config.browser_executable + if browser_executable: + # Use configured browser path + browser = browser_executable + else: + # Auto-detect: prefer Chromium, fallback to Firefox + browser = CHROMIUM or FIREFOX + if not browser: + raise RuntimeError('No browser found. Install Chromium or Firefox, or configure browser_executable.') + + # Get browser args and user agent from config + browser_args = config.browser_args or '["--no-sandbox"]' + user_agent = config.user_agent or DOWNLOAD_USER_AGENT + cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority. str(SINGLE_FILE_BIN), - '--browser-executable-path', CHROMIUM, - '--browser-args', '["--no-sandbox"]', - '--user-agent', DOWNLOAD_USER_AGENT, + '--browser-executable-path', str(browser), + '--browser-args', browser_args, + '--user-agent', user_agent, '--dump-content', '--load-deferred-images-dispatch-scroll-event', download.url, @@ -233,17 +251,7 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive: file_group.title = file_group.a_text = title or archive.file_group.title file_group.d_text = contents - # Store paths as filenames only (relative to directory), resolved via file_group.resolve_path() - file_group.data = { - 'id': archive.id, - 'domain': archive.domain, - 'readability_json_path': archive.readability_json_path.name if archive.readability_json_path else None, - 'readability_path': archive.readability_path.name if archive.readability_path else None, - 'readability_txt_path': archive.readability_txt_path.name if archive.readability_txt_path else None, - 'screenshot_path': archive.screenshot_path.name if archive.screenshot_path else None, - 'singlefile_path': archive.singlefile_path.name if archive.singlefile_path else None, - 'info_json_path': archive.info_json_path.name if archive.info_json_path else None, - } + archive.rebuild_data() return archive except Exception as e: diff --git a/modules/archive/api.py b/modules/archive/api.py index 6cd9a876..fc161ace 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -7,12 +7,14 @@ from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi.api_utils import json_response, api_app +from wrolpi.cmd import get_installed_browsers from wrolpi.common import logger, wrol_mode_check, api_param_limiter, TRACE_LEVEL from wrolpi.db import get_db_session from wrolpi.errors import ValidationError from wrolpi.events import Events from wrolpi.schema import JSONErrorResponse from wrolpi.switches import register_switch_handler, ActivateSwitchMethod +from wrolpi.vars import DOCKERIZED from . import lib, schema NAME = 'archive' @@ -90,6 +92,28 @@ async def post_file_format(_: Request, body: schema.ArchiveFileFormatRequest): ), status=HTTPStatus.BAD_REQUEST) +@archive_bp.get('/browsers') +@openapi.definition( + summary='Get list of installed browsers that can be used for archiving.', + description='Returns browsers installed on the system. Only available on native deployments (not Docker).', +) +async def get_browsers(_: Request): + """Get list of installed browsers for SingleFile.""" + if DOCKERIZED: + # Browser configuration is not available in Docker + return json_response(dict( + browsers=[], + available=False, + message='Browser configuration is not available in Docker deployments.', + )) + + browsers = get_installed_browsers() + return json_response(dict( + browsers=browsers, + available=True, + )) + + @archive_bp.get('/upload') @openapi.definition( summary='A message to confirm to the user that they have the correct upload URL.' diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 9a7cd4db..243d664c 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -22,7 +22,7 @@ from wrolpi.cmd import READABILITY_BIN, run_command from wrolpi.collections import Collection from wrolpi.common import get_media_directory, get_relative_to_media_directory, logger, extract_domain, \ escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \ - get_wrolpi_config, html_screenshot, ConfigFile + get_wrolpi_config, html_screenshot, ConfigFile, trim_file_name from wrolpi.dates import now, Seconds from wrolpi.db import get_db_session, get_db_curs from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime @@ -218,12 +218,26 @@ class ArchiveDownloaderConfigValidator: """Validator for archives_downloader.yaml config.""" file_name_format: str version: int = 0 + # Browser settings (only used on native deployments, not Docker) + browser_executable: str = None # None means auto-detect, or absolute path + browser_args: str = '["--no-sandbox"]' # JSON array of browser arguments + user_agent: str = None # None means use system default def __post_init__(self): # Validate file_name_format ends with .%(ext)s (like video format) if not self.file_name_format.endswith('.%(ext)s'): raise ValueError('file_name_format must end with .%(ext)s') + # Validate browser_args is valid JSON array + if self.browser_args: + import json + try: + args = json.loads(self.browser_args) + if not isinstance(args, list): + raise ValueError('browser_args must be a JSON array') + except json.JSONDecodeError as e: + raise ValueError(f'browser_args must be valid JSON: {e}') + class ArchiveDownloaderConfig(ConfigFile): """ @@ -235,6 +249,9 @@ class ArchiveDownloaderConfig(ConfigFile): Format: version: 0 file_name_format: '%(download_date)s_%(title)s.%(ext)s' + browser_executable: null # null for auto-detect, or absolute path + browser_args: '["--no-sandbox"]' # JSON array of browser arguments + user_agent: null # null for system default Variables available in file_name_format: - %(title)s - Page title (extracted from HTML) @@ -248,18 +265,41 @@ class ArchiveDownloaderConfig(ConfigFile): Subdirectories can be included in the format, e.g.: '%(download_year)s/%(download_datetime)s_%(title)s.%(ext)s' + + Browser settings (only used on native deployments, not Docker): + - browser_executable: Path to browser or null for auto-detect + - browser_args: JSON array of arguments passed to browser + - user_agent: Custom user agent string or null for default """ file_name = 'archives_downloader.yaml' validator = ArchiveDownloaderConfigValidator default_config = dict( version=0, file_name_format='%(download_datetime)s_%(title)s.%(ext)s', + browser_executable=None, + browser_args='["--no-sandbox"]', + user_agent=None, ) @property def file_name_format(self) -> str: return self._config['file_name_format'] + @property + def browser_executable(self) -> Optional[str]: + """Get the configured browser executable path, or None for auto-detect.""" + return self._config.get('browser_executable') + + @property + def browser_args(self) -> str: + """Get the browser arguments as a JSON string.""" + return self._config.get('browser_args', '["--no-sandbox"]') + + @property + def user_agent(self) -> Optional[str]: + """Get the custom user agent, or None for system default.""" + return self._config.get('user_agent') + def import_config(self, file: pathlib.Path = None, send_events=False): super().import_config(file, send_events) self.successful_import = True @@ -314,11 +354,20 @@ def format_archive_filename( ) try: - return template % variables + result = template % variables + # Trim the filename portion if it's too long (may include subdirectory paths) + if '/' in result: + # Has subdirectories - only trim the filename part + parts = result.rsplit('/', 1) + subdir = parts[0] + filename = trim_file_name(parts[1]) + return f'{subdir}/{filename}' + else: + return trim_file_name(result) except KeyError as e: logger.error(f'Invalid variable in archive file_name_format: {e}') # Fallback to default format - return f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html' + return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html') def preview_archive_filename(file_name_format: str) -> str: @@ -356,6 +405,69 @@ def preview_archive_filename(file_name_format: str) -> str: raise RuntimeError(f'Invalid format: {e}') +def format_archive_filename_from_archive( + archive: Archive, + file_name_format: str = None, +) -> str: + """Single source of truth for archive filenames when reorganizing. + + Formats an archive filename using the configured file_name_format template. + Used by collection reorganization to compute new file paths. + + Args: + archive: Archive model with metadata + file_name_format: Optional override, defaults to config value + + Returns: + Formatted filename (may include subdirectory path like "2025/My Article.html") + """ + config = get_archive_downloader_config() + template = file_name_format or config.file_name_format + + file_group = archive.file_group + + # Get download date from file_group + download_date = file_group.download_datetime or file_group.published_datetime or now() + + # Get title + title = file_group.title or 'untitled' + + # Get domain from URL + domain = '' + if file_group.url: + try: + domain = extract_domain(file_group.url) + except Exception: + pass + + variables = dict( + title=escape_file_name(title) if title else 'untitled', + domain=domain, + download_datetime=archive_strftime(download_date), + download_date=download_date.strftime('%Y-%m-%d'), + download_year=str(download_date.year), + download_month=f'{download_date.month:02d}', + download_day=f'{download_date.day:02d}', + ext='html', + ) + + try: + result = template % variables + # Trim the filename portion if it's too long (may include subdirectory paths) + if '/' in result: + # Has subdirectories - only trim the filename part + parts = result.rsplit('/', 1) + subdir = parts[0] + filename = trim_file_name(parts[1]) + return f'{subdir}/{filename}' + else: + return trim_file_name(result) + except KeyError as e: + logger.error(f'Invalid variable in archive file_name_format: {e}') + # Fallback to default format + return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html') + + @dataclass class ArchiveFiles: """Every Archive will have some of these files.""" @@ -416,17 +528,18 @@ def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.P full_path.parent.mkdir(parents=True, exist_ok=True) singlefile_path = full_path stem = full_path.stem - readability_path = full_path.parent / f'{stem}.readability.html' - readability_txt_path = full_path.parent / f'{stem}.readability.txt' - readability_json_path = full_path.parent / f'{stem}.readability.json' - screenshot_path = full_path.parent / f'{stem}.png' + parent_dir = full_path.parent + readability_path = trim_file_name(parent_dir / f'{stem}.readability.html') + readability_txt_path = trim_file_name(parent_dir / f'{stem}.readability.txt') + readability_json_path = trim_file_name(parent_dir / f'{stem}.readability.json') + screenshot_path = trim_file_name(parent_dir / f'{stem}.png') else: singlefile_path = directory / formatted_filename stem = singlefile_path.stem - readability_path = directory / f'{stem}.readability.html' - readability_txt_path = directory / f'{stem}.readability.txt' - readability_json_path = directory / f'{stem}.readability.json' - screenshot_path = directory / f'{stem}.png' + readability_path = trim_file_name(directory / f'{stem}.readability.html') + readability_txt_path = trim_file_name(directory / f'{stem}.readability.txt') + readability_json_path = trim_file_name(directory / f'{stem}.readability.json') + screenshot_path = trim_file_name(directory / f'{stem}.png') paths = (singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path) @@ -592,6 +705,15 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr archive.file_group.download_datetime = now() archive.url = url archive.collection = get_or_create_domain_collection(session, url) + + # Update collection's file_format to track the format used for this file + config = get_archive_downloader_config() + if archive.collection and config.file_name_format: + if not archive.collection.file_format: + # Set initial file_format on collection + archive.collection.file_format = config.file_name_format + save_domains_config.activate_switch() + archive.flush() return archive diff --git a/modules/archive/models.py b/modules/archive/models.py index acb292af..87e16335 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -351,6 +351,23 @@ class Archive(Base, ModelHelper): if metadata.modified_datetime: self.file_group.published_modified_datetime = metadata.modified_datetime + def rebuild_data(self): + """Rebuild the data field with current file paths. + + Called during validate() to ensure data field reflects actual filenames + after reorganization or other file operations. + """ + self.file_group.data = { + 'id': self.id, + 'domain': self.domain, + 'readability_json_path': self.readability_json_path.name if self.readability_json_path else None, + 'readability_path': self.readability_path.name if self.readability_path else None, + 'readability_txt_path': self.readability_txt_path.name if self.readability_txt_path else None, + 'screenshot_path': self.screenshot_path.name if self.screenshot_path else None, + 'singlefile_path': self.singlefile_path.name if self.singlefile_path else None, + 'info_json_path': self.info_json_path.name if self.info_json_path else None, + } + def validate(self): """Fill in any missing data about this Archive from its files.""" try: @@ -359,6 +376,7 @@ class Archive(Base, ModelHelper): self.apply_domain() self.apply_singlefile_title() self.apply_metadata() + self.rebuild_data() except Exception as e: logger.warning(f'Unable to validate {self}', exc_info=e) if PYTEST: diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index bed82095..d35feb1d 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -342,18 +342,9 @@ async def test_archive_generate_screenshot(test_session, async_client, archive_f assert original_screenshot_path is not None, 'Archive should have a screenshot' assert original_screenshot_path.is_file(), 'Screenshot file should exist' - # Simulate the bug: archive has screenshot file but data['screenshot_path'] is not set - # (This could happen if the archive was created before set_screenshot was implemented) - if archive.file_group.data and 'screenshot_path' in archive.file_group.data: - data = dict(archive.file_group.data) - del data['screenshot_path'] - archive.file_group.data = data - test_session.commit() - test_session.expire_all() - archive = test_session.query(Archive).filter_by(id=archive.id).one() - # Verify data was cleared but file still exists - assert 'screenshot_path' not in (archive.file_group.data or {}), 'screenshot_path should not be in data yet' - assert archive.screenshot_path is not None, 'But screenshot file should still exist' + # Note: Previously this test simulated a bug where data['screenshot_path'] could be missing + # while the file existed. This scenario is now auto-fixed by Archive.validate() which calls + # rebuild_data(), so the simulation is no longer necessary. request, response = await async_client.post(f'/api/archive/{archive.id}/generate_screenshot') assert response.status_code == HTTPStatus.OK @@ -364,11 +355,11 @@ async def test_archive_generate_screenshot(test_session, async_client, archive_f assert archive.screenshot_path == original_screenshot_path, 'Screenshot path should be unchanged' assert archive.screenshot_path.is_file(), 'Screenshot file should still exist' assert archive.screenshot_file is not None, 'Screenshot file should still be tracked' - # IMPORTANT: Also verify FileGroup.data was updated (this catches the bug where only files was updated) + # IMPORTANT: Also verify FileGroup.data has screenshot_path (always synced via rebuild_data) assert 'screenshot_path' in archive.file_group.data, \ - 'screenshot_path should be in FileGroup.data after ensuring tracking' - assert str(archive.file_group.data['screenshot_path']) == str(original_screenshot_path), \ - 'screenshot_path in data should match the file path' + 'screenshot_path should be in FileGroup.data' + assert archive.file_group.data['screenshot_path'] == original_screenshot_path.name, \ + 'screenshot_path in data should be filename only' # Test error case: Archive has no singlefile archive_no_singlefile = archive_factory('example.com', 'https://example.com/no-singlefile', screenshot=False) @@ -428,3 +419,28 @@ async def test_archive_file_format_preview(async_client): request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data)) assert response.status_code == HTTPStatus.BAD_REQUEST assert 'error' in response.json + + +@pytest.mark.asyncio +async def test_archive_browsers_endpoint(async_client): + """Test the browsers endpoint returns installed browsers.""" + # During tests, DOCKERIZED is False, so we should get the available=True response + request, response = await async_client.get('/api/archive/browsers') + assert response.status_code == HTTPStatus.OK + assert 'browsers' in response.json + assert 'available' in response.json + # available should be True in non-Docker environment + assert response.json['available'] is True + # browsers is a list + assert isinstance(response.json['browsers'], list) + + +@pytest.mark.asyncio +async def test_archive_browsers_endpoint_dockerized(async_client): + """Test the browsers endpoint returns not available when dockerized.""" + with patch('modules.archive.api.DOCKERIZED', True): + request, response = await async_client.get('/api/archive/browsers') + assert response.status_code == HTTPStatus.OK + assert response.json['browsers'] == [] + assert response.json['available'] is False + assert 'message' in response.json diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 077e23bd..1e6446ce 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -365,14 +365,17 @@ async def test_get_title_from_html(test_directory, test_session, fake_now, async @skip_circleci def test_get_new_archive_files_length(test_directory, fake_now): - """Archive titles are truncated to fit file system length limit. (255-character limit for most file systems)""" + """Archive titles are truncated to fit file system BYTE limit (200 bytes - see MAXIMUM_FILE_BYTES).""" + from wrolpi.common import MAXIMUM_FILE_BYTES + fake_now(datetime(2001, 1, 1)) archive_files = get_new_archive_files('https://example.com', 'a' * 500) - assert len(str(archive_files.singlefile.name)) == 225 - assert len(str(archive_files.readability.name)) == 237 - assert len(str(archive_files.readability_txt.name)) == 236 - assert len(str(archive_files.readability_json.name)) == 237 - assert len(str(archive_files.screenshot.name)) == 224 + # All filenames should be <= MAXIMUM_FILE_BYTES (bytes, not characters) + assert len(str(archive_files.singlefile.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES + assert len(str(archive_files.readability.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES + assert len(str(archive_files.readability_txt.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES + assert len(str(archive_files.readability_json.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES + assert len(str(archive_files.screenshot.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES @skip_circleci @@ -1289,6 +1292,26 @@ def test_archive_downloader_config_validation(): with pytest.raises(ValueError, match='must end with'): ArchiveDownloaderConfigValidator(file_name_format='%(download_date)s_%(title)s') + # Valid browser_args (JSON array) + ArchiveDownloaderConfigValidator( + file_name_format='%(title)s.%(ext)s', + browser_args='["--no-sandbox", "--headless"]' + ) + + # Invalid browser_args (not valid JSON) + with pytest.raises(ValueError, match='must be valid JSON'): + ArchiveDownloaderConfigValidator( + file_name_format='%(title)s.%(ext)s', + browser_args='not valid json' + ) + + # Invalid browser_args (not an array) + with pytest.raises(ValueError, match='must be a JSON array'): + ArchiveDownloaderConfigValidator( + file_name_format='%(title)s.%(ext)s', + browser_args='{"key": "value"}' + ) + def test_format_archive_filename_default(test_directory, fake_now): """format_archive_filename uses default format.""" diff --git a/modules/archive/test/test_models.py b/modules/archive/test/test_models.py index 4b9985e2..a3a0097c 100644 --- a/modules/archive/test/test_models.py +++ b/modules/archive/test/test_models.py @@ -128,3 +128,53 @@ async def test_set_screenshot(async_client, test_session, archive_factory, image # Verify no screenshot was set assert archive_no_screenshot.screenshot_path is None + + +@pytest.mark.asyncio +async def test_archive_validate_rebuilds_data_after_filename_change(async_client, test_session, test_directory, + archive_factory): + """Archive.validate() should rebuild data field when filenames change. + + This tests the bug where reorganization updates files[] but not data{}. + The data field contains cached path names that must stay in sync. + + The bug scenario: files are renamed during reorganization (e.g., fixing separator + from '___' to '_'), files[] is updated but data{} is not. The UI then uses + stale data.singlefile_path and gets 404 errors. + """ + # Create an archive using the factory + archive = archive_factory(domain='test.com', url='https://test.com/article') + test_session.commit() + + # Verify initial data field has singlefile_path + assert archive.file_group.data is not None + original_singlefile = archive.file_group.data.get('singlefile_path') + assert original_singlefile is not None + + # Simulate what reorganization does: update files[] but not data{} + # (This mimics _bulk_update_file_groups_reorganize behavior) + # Use a realistic rename: change the title part of the filename + # Original: '2000-01-01-07-00-01_NA.html' -> '2000-01-01-07-00-01_New-Title.html' + new_filename = original_singlefile.replace('_NA.html', '_New-Title.html') + old_files = archive.file_group.files + new_files = [] + for f in old_files: + new_f = dict(f) + if f['path'].endswith('.html') and not f['path'].endswith('.readability.html'): + new_f['path'] = new_filename + new_files.append(new_f) + archive.file_group.files = new_files + archive.file_group.primary_path = archive.file_group.directory / new_filename + + # Rename the actual file on disk to match the new filename + old_path = archive.file_group.directory / original_singlefile + new_path = archive.file_group.directory / new_filename + old_path.rename(new_path) + + # data field still has OLD filename (this is the bug) + assert archive.file_group.data['singlefile_path'] == original_singlefile + + # After validate(), data field should have NEW filename + archive.validate() + + assert archive.file_group.data['singlefile_path'] == new_filename diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index 70ac71fe..ab8a109a 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -260,6 +260,9 @@ async def test_channel_post_directory(test_session, async_client, test_directory async def test_channel_by_id(test_session, async_client, simple_channel, simple_video): request, response = await async_client.get(f'/api/videos/channels/{simple_channel.id}') assert response.status_code == HTTPStatus.OK + channel = response.json['channel'] + # needs_reorganization should be included for UI to display reorganization status + assert 'needs_reorganization' in channel @pytest.mark.asyncio diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index f560d2b2..f3516cb3 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -139,7 +139,10 @@ def convert_wrolpi_filename_format(filename_format: str) -> str: def preview_filename(filename_format: str) -> str: - """Return an example video file name formatted using the provided filename format. + """Return an example video file name using simple string formatting. + + This is a lightweight implementation that doesn't require YoutubeDL instantiation, + which is expensive (~1500 extractors loaded, regexes compiled). @raise RuntimeError: If the format is invalid. """ @@ -149,21 +152,31 @@ def preview_filename(filename_format: str) -> str: # Convert WROLPi-specific variables to yt-dlp syntax converted_format = convert_wrolpi_filename_format(filename_format) - options = get_videos_downloader_config().yt_dlp_options - options['outtmpl'] = converted_format - ydl = YoutubeDL(copy.deepcopy(options)) - entry = dict( + sample_date = now() + variables = dict( uploader='WROLPi', - timestamp=int(now().timestamp()), - upload_date=now().strftime('%Y%m%d'), + channel='WROLPi', + upload_date=sample_date.strftime('%Y%m%d'), id='Qz-FuenRylQ', title='The title of the video', ext='mp4', description='A description of the video', + timestamp=int(sample_date.timestamp()), ) - filename = prepare_filename(entry, ydl=ydl).lstrip('/') - return filename + # Handle yt-dlp's date extraction syntax %(upload_date>%Y)s and %(upload_date>%m)s + # Replace these with their values before standard Python % formatting + def replace_date_format(match): + date_format = match.group(1) + return sample_date.strftime(date_format) + + converted_format = re.sub(r'%\(upload_date>([^)]+)\)s', replace_date_format, converted_format) + + try: + filename = converted_format % variables + return filename.lstrip('/') + except KeyError as e: + raise RuntimeError(f'Invalid variable: {e}') @cached_multiprocessing_result @@ -545,6 +558,17 @@ class VideoDownloader(Downloader, ABC): video_id, video_info_json_path = video.id, video.info_json_path if video_info_json_path and (new_info_json := video.clean_info_json()): video.replace_info_json(new_info_json, clean=False) + + # Update channel's collection's file_format to track the format used + if channel_id: + from modules.videos.models import Channel + from modules.videos.lib import save_channels_config + channel_obj = session.query(Channel).get(channel_id) + if channel_obj and channel_obj.collection and not channel_obj.collection.file_format: + video_config = get_videos_downloader_config() + channel_obj.collection.file_format = video_config.file_name_format + save_channels_config.activate_switch() + session.commit() with get_db_session(commit=True) as session: diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 4129b12e..aba9b358 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -16,7 +16,7 @@ from modules.videos.models import Video from wrolpi import dates, flags from wrolpi.captions import extract_captions from wrolpi.common import ConfigFile, extract_domain, logger, \ - escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config + escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config, trim_file_name from wrolpi.dates import Seconds, from_timestamp from wrolpi.db import get_db_curs, get_db_session from wrolpi.downloader import Download, download_manager @@ -948,3 +948,85 @@ def browser_profile_to_yt_dlp_arg(profile: pathlib.Path) -> str: """Takes a Path and returns a string that can be used as a yt-dlp `--cookies-from-browser` argument.""" *_, browser, profile = str(profile).split('/') return f'{browser}:{profile}' + + +def format_video_filename( + video: Video, + file_name_format: str = None, +) -> str: + """Single source of truth for video filenames. + + Formats a video filename using the configured file_name_format template. + Used by both the video downloader (initial download) and collection reorganization. + + Args: + video: Video model with metadata (upload_date, uploader, title, source_id) + file_name_format: Optional override, defaults to config value + + Returns: + Formatted filename (may include subdirectory path like "2025/01/video.mp4") + + Example: + With format '%(upload_year)s/%(upload_month)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s': + Returns '2025/01/WROLPi_20250115_abc123_My Video Title.mp4' + """ + from modules.videos.downloader import convert_wrolpi_filename_format + + config = get_videos_downloader_config() + template = file_name_format or config.file_name_format + + # Get metadata from video + file_group = video.file_group + info_json = video.get_info_json() or {} + + # Extract upload date info + upload_date = file_group.published_datetime + if not upload_date and info_json.get('upload_date'): + upload_date = dates.strpdate(info_json['upload_date']) + + # Build variables dict matching yt-dlp's expected variables + # Use info_json if available, fall back to derived values + uploader = info_json.get('uploader') or info_json.get('channel') or '' + title = file_group.title or info_json.get('title') or info_json.get('fulltitle') or 'untitled' + source_id = video.source_id or info_json.get('id') or '' + + # Get video file extension + video_path = video.video_path + ext = video_path.suffix.lstrip('.') if video_path else config.merge_output_format or 'mp4' + + variables = dict( + uploader=escape_file_name(uploader) if uploader else '', + channel=escape_file_name(uploader) if uploader else '', # Alias + upload_date=upload_date.strftime('%Y%m%d') if upload_date else '', + id=source_id, + title=escape_file_name(title) if title else 'untitled', + ext=ext, + # WROLPi-specific variables for subdirectory organization + upload_year=str(upload_date.year) if upload_date else '', + upload_month=f'{upload_date.month:02d}' if upload_date else '', + ) + + # Convert WROLPi-specific variables to yt-dlp syntax then apply + # (This handles %(upload_year)s -> %(upload_date>%Y)s conversion) + converted_template = convert_wrolpi_filename_format(template) + + # For the converted variables (like %(upload_date>%Y)s), we need to add them too + variables['upload_date>%Y'] = str(upload_date.year) if upload_date else '' + variables['upload_date>%m'] = f'{upload_date.month:02d}' if upload_date else '' + + try: + result = converted_template % variables + # Trim the filename portion if it's too long (may include subdirectory paths) + if '/' in result: + # Has subdirectories - only trim the filename part + parts = result.rsplit('/', 1) + subdir = parts[0] + filename = trim_file_name(parts[1]) + return f'{subdir}/{filename}' + else: + return trim_file_name(result) + except KeyError as e: + logger.error(f'Invalid variable in video file_name_format: {e}') + # Fallback to a simple format + fallback_title = escape_file_name(title) if title else 'untitled' + return trim_file_name(f'{fallback_title}.{ext}') diff --git a/modules/videos/models.py b/modules/videos/models.py index 75fe742f..c0972ed9 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -785,6 +785,9 @@ class Channel(ModelHelper, Base): tag_name=self.tag_name, url=self.url or None, ) + # Include file_format from the underlying Collection + if self.collection and self.collection.file_format: + config['file_format'] = self.collection.file_format return config @classmethod @@ -860,6 +863,7 @@ class Channel(ModelHelper, Base): d['rss_url'] = self.get_rss_url() d['directory'] = \ self.directory.relative_to(get_media_directory()) if self.directory else None + d['needs_reorganization'] = self.collection.needs_reorganization if self.collection else False if with_statistics: d['statistics'] = self.get_statistics() if with_downloads: diff --git a/modules/videos/test/test_channels_config.py b/modules/videos/test/test_channels_config.py index 1952e274..dbd53ca4 100644 --- a/modules/videos/test/test_channels_config.py +++ b/modules/videos/test/test_channels_config.py @@ -280,3 +280,45 @@ class TestChannelsConfigEdgeCases: # Directory should be relative, not absolute assert not config['directory'].startswith('/'), \ f"Expected relative path but got absolute: {config['directory']}" + + async def test_channel_config_view_includes_file_format(self, test_session, test_directory, async_client, + test_channels_config, channel_factory): + """Channel.config_view() should include file_format from underlying Collection.""" + channel = channel_factory() + channel.collection.file_format = '%(uploader)s_%(id)s_%(title)s.%(ext)s' + test_session.commit() + + config = channel.config_view() + assert config['file_format'] == '%(uploader)s_%(id)s_%(title)s.%(ext)s' + + async def test_channel_file_format_round_trip(self, test_session, test_directory, async_client, + test_channels_config, channel_factory): + """File format should survive config dump and import cycle.""" + config = get_channels_config() + config_path = config.get_file() + + # Create channel with file_format + channel = channel_factory(name='Format Test') + channel.collection.file_format = '%(upload_date)s_%(title)s.%(ext)s' + test_session.commit() + + # Dump to config + config.dump_config() + + # Verify file_format in YAML + with open(config_path) as f: + config_data = yaml.safe_load(f) + channel_config = next(c for c in config_data['channels'] if c['name'] == 'Format Test') + assert channel_config['file_format'] == '%(upload_date)s_%(title)s.%(ext)s' + + # Clear the file_format to simulate factory reset + channel.collection.file_format = None + test_session.commit() + + # Re-import + config.initialize() + config.import_config() + + # Verify file_format restored + test_session.refresh(channel) + assert channel.collection.file_format == '%(upload_date)s_%(title)s.%(ext)s' diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index fa0185b1..60f103a2 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -8,7 +8,8 @@ from unittest import mock import pytest from modules.videos.downloader import VideoDownloader, \ - get_or_create_channel, channel_downloader, video_downloader, preview_filename + get_or_create_channel, channel_downloader, video_downloader, preview_filename, \ + prepare_filename, convert_wrolpi_filename_format from modules.videos.lib import get_videos_downloader_config from modules.videos.models import Channel, Video from wrolpi.conftest import test_directory, await_switches @@ -573,6 +574,54 @@ def test_preview_filename_with_upload_year(test_directory, fake_now): == '2000/WROLPi_20000101_The title of the video.mp4' +def test_preview_filename_matches_youtubedl(test_directory, fake_now): + """Ensure our lightweight preview_filename() matches YoutubeDL's actual output. + + This test validates that our simple string formatting produces identical + results to YoutubeDL's prepare_filename(). If yt-dlp changes behavior, + this test will catch it. + """ + from copy import deepcopy + from yt_dlp import YoutubeDL + from wrolpi.dates import now + + test_formats = [ + '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s', + '%(upload_date)s_%(id)s_%(title)s.%(ext)s', + '%(upload_year)s/%(upload_month)s/%(title)s.%(ext)s', + '%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s', + '%(channel)s/%(id)s.%(ext)s', + ] + + # Sample entry used by both methods - must match preview_filename's variables + sample_date = now() + entry = dict( + uploader='WROLPi', + channel='WROLPi', + timestamp=int(sample_date.timestamp()), + upload_date=sample_date.strftime('%Y%m%d'), + id='Qz-FuenRylQ', + title='The title of the video', + ext='mp4', + description='A description of the video', + ) + + for fmt in test_formats: + # Get result from our lightweight implementation + lightweight_result = preview_filename(fmt) + + # Get result from actual YoutubeDL + converted_format = convert_wrolpi_filename_format(fmt) + options = get_videos_downloader_config().yt_dlp_options + options = deepcopy(options) + options['outtmpl'] = converted_format + ydl = YoutubeDL(options) + ytdlp_result = prepare_filename(entry, ydl=ydl).lstrip('/') + + assert lightweight_result == ytdlp_result, \ + f"Mismatch for format '{fmt}':\n lightweight: {lightweight_result}\n ytdlp: {ytdlp_result}" + + def test_normalize_video_file_names(test_directory, video_download_manager): """VideoDownloader renames caption files that do not share the video file's stem.""" downloader = VideoDownloader() diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index e4312752..ef1d7eff 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -326,3 +326,32 @@ def test_video_location(test_session, video_factory, channel_factory, async_clie channel = channel_factory(name='ChannelName') video2 = video_factory(title='vid2', channel_id=channel.id) assert video2.location == '/videos/channel/1/video/2' + + +def test_format_video_filename_without_upload_date(test_session, video_factory, test_videos_downloader_config, caplog): + """Videos without upload_date should not cause KeyError when format uses upload_year.""" + import logging + from modules.videos.lib import format_video_filename + + # Create video without upload_date + video = video_factory(title='Test Video', with_video_file=True, with_info_json={'id': 'abc123'}) + assert video.file_group.published_datetime is None + + # Use a format that includes upload_year subdirectory + template = '%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s' + + # Should not raise KeyError and should not log an error + with caplog.at_level(logging.ERROR): + result = format_video_filename(video, template) + + # Should not have logged any errors (fallback was not used) + assert 'Invalid variable in video file_name_format' not in caplog.text + + # Should return a properly formatted filename with the template structure + # (empty year and date components produce a path with subdirectory) + assert result.endswith('.mp4') + assert 'Test Video' in result + # The result should follow the template format, not the fallback format + # Template result: '{upload_year}/{uploader}_{upload_date}_{id}_{title}.mp4' + # With empty year/date: '/__Test Video_Test Video.mp4' (has a slash for subdirectory) + assert '/' in result # Confirms template was used (has subdirectory), not fallback diff --git a/pytest.ini b/pytest.ini index 7a28bf61..aa680dee 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,5 +5,5 @@ testpaths = modules asyncio_default_fixture_loop_scope = function # Do not remove timeout - tests should not run forever -timeout = 20 +timeout = 30 timeout_method = signal diff --git a/requirements.txt b/requirements.txt index c07dc247..bf559b0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,4 +30,4 @@ srt==3.5.3 tqdm==4.67.1 vininfo[cli]==1.8.0 webvtt-py==0.5.1 -yt-dlp==2025.12.08 +yt-dlp==2026.01.29 diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py index 0bb74f96..5f156161 100644 --- a/wrolpi/cmd.py +++ b/wrolpi/cmd.py @@ -79,11 +79,54 @@ CHROMIUM = which('chromium-browser', 'chromium', '/usr/bin/chromium-browser', # rpi os '/usr/bin/chromium', # debian ) +FIREFOX = which('firefox', 'firefox-esr', + '/usr/bin/firefox', + '/usr/bin/firefox-esr', + ) READABILITY_BIN = which('readability-extractor', '/usr/bin/readability-extractor', # rpi os '/usr/local/bin/readability-extractor', # debian ) +# Known browser definitions for SingleFile +KNOWN_BROWSERS = { + 'chromium': { + 'paths': ['chromium-browser', 'chromium', '/usr/bin/chromium-browser', '/usr/bin/chromium'], + 'name': 'Chromium', + }, + 'firefox': { + 'paths': ['firefox', 'firefox-esr', '/usr/bin/firefox', '/usr/bin/firefox-esr'], + 'name': 'Firefox', + }, + 'brave': { + 'paths': ['brave-browser', '/usr/bin/brave-browser'], + 'name': 'Brave', + } +} + + +def get_installed_browsers() -> list: + """ + Detect browsers installed on the system that can be used with SingleFile. + + Returns a list of dicts with: + - key: The browser identifier (e.g., 'chromium', 'firefox') + - name: Human-readable name (e.g., 'Chromium', 'Firefox') + - path: Absolute path to the executable + """ + browsers = [] + + for key, info in KNOWN_BROWSERS.items(): + path = which(*info['paths']) + if path: + browsers.append({ + 'key': key, + 'name': info['name'], + 'path': str(path), + }) + + return browsers + # Videos YT_DLP_BIN = which( '/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location diff --git a/wrolpi/collections/api.py b/wrolpi/collections/api.py index 09e3f610..7c497ca8 100644 --- a/wrolpi/collections/api.py +++ b/wrolpi/collections/api.py @@ -266,3 +266,224 @@ async def search_collections_endpoint(request: Request, body: schema.CollectionS 'collections': collections, 'totals': {'collections': len(collections)} }) + + +@collection_bp.get('//reorganize/preview') +@openapi.summary('Preview collection reorganization') +@openapi.response(HTTPStatus.OK, schema.ReorganizationPreviewResponse, description="Reorganization preview") +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) +async def preview_reorganization_endpoint(request: Request, collection_id: int): + """ + Get preview with sample of files to be renamed. + + Returns a preview of what would happen if the collection is reorganized. + This is useful for showing users what files would move before they commit. + + Returns: + - collection_id: The collection ID + - collection_name: The collection name + - total_files: Total FileGroups in collection + - files_needing_move: Count needing reorganization + - sample_moves: ~10 example moves [{old_path, new_path}, ...] + - new_file_format: The format template that will be applied + - current_file_format: The format currently stored on collection + """ + from wrolpi.collections.reorganize import get_reorganization_preview + + session = request.ctx.session + try: + # Use exact_count=True for individual collection preview to get accurate count + preview = get_reorganization_preview(collection_id, session, exact_count=True) + return json_response({ + 'collection_id': preview.collection_id, + 'collection_name': preview.collection_name, + 'total_files': preview.total_files, + 'files_needing_move': preview.files_needing_move, + 'sample_moves': preview.sample_moves, + 'new_file_format': preview.new_file_format, + 'current_file_format': preview.current_file_format, + }) + except UnknownCollection as e: + return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND) + except ValueError as e: + return json_response({'error': str(e)}, status=HTTPStatus.BAD_REQUEST) + + +@collection_bp.post('//reorganize') +@openapi.summary('Execute collection reorganization') +@openapi.response(HTTPStatus.OK, schema.ReorganizationExecuteResponse, description="Reorganization started") +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) +@wrol_mode_check +async def execute_reorganization_endpoint(request: Request, collection_id: int): + """ + Execute reorganization. + + Moves files to match the current file_name_format configuration. + The operation runs in the background via FileWorker. + + Returns: + - job_id: For tracking progress via the status endpoint + - message: Status message + """ + from wrolpi.collections.reorganize import execute_reorganization + + session = request.ctx.session + try: + job_id = execute_reorganization(collection_id, session) + if job_id: + return json_response({ + 'job_id': job_id, + 'message': 'Reorganization started' + }) + else: + return json_response({ + 'job_id': '', + 'message': 'No files need reorganization' + }) + except UnknownCollection as e: + return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND) + except ValueError as e: + return json_response({'error': str(e)}, status=HTTPStatus.BAD_REQUEST) + + +@collection_bp.get('//reorganize/status/') +@openapi.summary('Get reorganization status') +@openapi.response(HTTPStatus.OK, schema.ReorganizationStatusResponse, description="Reorganization status") +async def get_reorganization_status_endpoint(_: Request, collection_id: int, job_id: str): + """ + Get status of in-flight reorganization. + + Returns: + - status: 'pending' | 'running' | 'complete' | 'failed' + - total: Total files to move + - completed: Files moved so far + - percent: Completion percentage + - error: Error message if failed + """ + from wrolpi.collections.reorganize import get_reorganization_status + + status = get_reorganization_status(job_id) + return json_response(status) + + +# ============================================================================ +# Batch Reorganization Endpoints +# ============================================================================ + + +@collection_bp.get('/reorganize/channels') +@openapi.summary('List channels needing batch reorganization') +@openapi.response(HTTPStatus.OK, schema.BatchReorganizationListResponse, + description="List of channels needing reorganization") +async def list_channels_needing_reorganization_endpoint(request: Request): + """ + List all channels that need reorganization because their file_name_format + differs from the current config. + + Returns: + - collections: List of channel info with sample moves + - total_collections: Total count needing reorganization + - total_files_needing_move: Sum of all files needing reorganization + - new_file_format: The config format that will be applied + """ + from wrolpi.collections.reorganize import get_collections_needing_reorganization + + session = request.ctx.session + result = get_collections_needing_reorganization('channel', session) + return json_response(result) + + +@collection_bp.get('/reorganize/domains') +@openapi.summary('List domains needing batch reorganization') +@openapi.response(HTTPStatus.OK, schema.BatchReorganizationListResponse, + description="List of domains needing reorganization") +async def list_domains_needing_reorganization_endpoint(request: Request): + """ + List all domains that need reorganization because their file_name_format + differs from the current config. + + Returns: + - collections: List of domain info with sample moves + - total_collections: Total count needing reorganization + - total_files_needing_move: Sum of all files needing reorganization + - new_file_format: The config format that will be applied + """ + from wrolpi.collections.reorganize import get_collections_needing_reorganization + + session = request.ctx.session + result = get_collections_needing_reorganization('domain', session) + return json_response(result) + + +@collection_bp.post('/reorganize/channels') +@openapi.summary('Execute batch reorganization of all channels') +@openapi.response(HTTPStatus.OK, schema.BatchReorganizationExecuteResponse, + description="Batch reorganization started") +@wrol_mode_check +async def execute_batch_reorganization_channels_endpoint(request: Request): + """ + Execute batch reorganization of all channels that need it. + + Processes channels sequentially. If any channel fails, the batch stops + and reports which channel failed. + + Returns: + - batch_job_id: For tracking progress via the batch status endpoint + - message: Status message + - collection_count: Number of channels to process + """ + from wrolpi.collections.reorganize import execute_batch_reorganization + + session = request.ctx.session + result = execute_batch_reorganization('channel', session) + return json_response(result) + + +@collection_bp.post('/reorganize/domains') +@openapi.summary('Execute batch reorganization of all domains') +@openapi.response(HTTPStatus.OK, schema.BatchReorganizationExecuteResponse, + description="Batch reorganization started") +@wrol_mode_check +async def execute_batch_reorganization_domains_endpoint(request: Request): + """ + Execute batch reorganization of all domains that need it. + + Processes domains sequentially. If any domain fails, the batch stops + and reports which domain failed. + + Returns: + - batch_job_id: For tracking progress via the batch status endpoint + - message: Status message + - collection_count: Number of domains to process + """ + from wrolpi.collections.reorganize import execute_batch_reorganization + + session = request.ctx.session + result = execute_batch_reorganization('domain', session) + return json_response(result) + + +@collection_bp.get('/reorganize/batch/status/') +@openapi.summary('Get batch reorganization status') +@openapi.response(HTTPStatus.OK, schema.BatchReorganizationStatusResponse, + description="Batch reorganization status") +async def get_batch_reorganization_status_endpoint(_: Request, batch_job_id: str): + """ + Get status of a batch reorganization job. + + Returns: + - status: 'pending' | 'running' | 'complete' | 'failed' | 'unknown' + - total_collections: Total collections in batch + - completed_collections: Number completed so far + - current_collection: Info about currently processing collection + - overall_percent: Overall completion percentage + - completed: List of completed collection info + - failed_collection: Info about failed collection (if any) + - error: Error message if failed + """ + from wrolpi.collections.reorganize import get_batch_reorganization_status + + status = get_batch_reorganization_status(batch_job_id) + return json_response(status) diff --git a/wrolpi/collections/errors.py b/wrolpi/collections/errors.py index 197f5951..d5a60452 100644 --- a/wrolpi/collections/errors.py +++ b/wrolpi/collections/errors.py @@ -1,9 +1,15 @@ from wrolpi.errors import APIError -__all__ = ['UnknownCollection'] +__all__ = ['UnknownCollection', 'ReorganizationConflict'] class UnknownCollection(APIError): """Cannot find Collection""" code = 'UNKNOWN_COLLECTION' summary = 'Cannot find Collection' + + +class ReorganizationConflict(APIError): + """Reorganization would cause filename conflicts""" + code = 'REORGANIZATION_CONFLICT' + summary = 'Reorganization would cause filename conflicts' diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index bb81f139..8aa985b8 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -90,6 +90,10 @@ class Collection(ModelHelper, Base): tag_id = Column(Integer, ForeignKey('tag.id')) tag = relationship('Tag', primaryjoin='Collection.tag_id==Tag.id') + # Stores the file_name_format used when files were last organized + # Used to detect when reorganization is needed after config changes + file_format = Column(String, nullable=True) + created_date = Column(DateTime, server_default=func.now(), nullable=False) # Columns updated by triggers (similar to Channel) @@ -119,6 +123,36 @@ class Collection(ModelHelper, Base): def tag_name(self) -> str | None: return self.tag.name if self.tag else None + @property + def needs_reorganization(self) -> bool: + """Returns True if collection's file_format differs from current config. + + When a user changes the file_name_format in their config, existing files + remain in their old locations. This property detects that mismatch so the + UI can offer to reorganize files. + + Returns True when: + - file_format is None (collection has never been synced to config) + - file_format differs from current config format + """ + current = self._get_current_file_format() + if not current: + return False + if not self.file_format: + # Never been reorganized - needs to sync to current config + return True + return current != self.file_format + + def _get_current_file_format(self) -> str | None: + """Get current file_name_format from config based on collection kind.""" + if self.kind == 'channel': + from modules.videos.lib import get_videos_downloader_config + return get_videos_downloader_config().file_name_format + elif self.kind == 'domain': + from modules.archive.lib import get_archive_downloader_config + return get_archive_downloader_config().file_name_format + return None + @property def location(self) -> str: """The href of the collection in the App.""" @@ -175,6 +209,9 @@ class Collection(ModelHelper, Base): config['tag_name'] = self.tag_name + if self.file_format: + config['file_format'] = self.file_format + # Include downloads if any exist if self.downloads: config['downloads'] = [ @@ -251,6 +288,7 @@ class Collection(ModelHelper, Base): description = data.get('description') directory = data.get('directory') tag_name = data.get('tag_name') + file_format = data.get('file_format') kind = (data.get('kind') or 'channel').strip() # Validate known kinds (forward-compatible: allow future values) if kind not in {'channel', 'domain'}: @@ -280,6 +318,7 @@ class Collection(ModelHelper, Base): collection.description = description collection.directory = directory collection.kind = kind + collection.file_format = file_format if tag_name: tag = Tag.get_by_name(session, tag_name) @@ -298,6 +337,7 @@ class Collection(ModelHelper, Base): description=description, directory=directory, kind=kind, + file_format=file_format, ) if tag_name: @@ -387,6 +427,7 @@ class Collection(ModelHelper, Base): description = data.get('description') directory = data.get('directory') tag_name = data.get('tag_name') + file_format = data.get('file_format') kind = (data.get('kind') or 'channel').strip() # Validate known kinds @@ -421,6 +462,7 @@ class Collection(ModelHelper, Base): collection.description = description collection.directory = directory collection.kind = kind + collection.file_format = file_format if tag_name: tag = tags_by_name.get(tag_name) @@ -439,6 +481,7 @@ class Collection(ModelHelper, Base): description=description, directory=directory, kind=kind, + file_format=file_format, ) if tag_name: @@ -824,6 +867,8 @@ class Collection(ModelHelper, Base): 'item_count': self.item_count, 'total_size': self.total_size, 'downloads': self.downloads, + 'file_format': self.file_format, + 'needs_reorganization': self.needs_reorganization, } def format_directory(self, tag_name: Optional[str]) -> pathlib.Path: @@ -898,13 +943,14 @@ class Collection(ModelHelper, Base): Absolute path to the collection's directory """ if self.directory: - return get_media_directory() / self.directory + # Return absolute path (joining with media_directory handles both relative and absolute paths) + return self.directory if self.directory.is_absolute() else get_media_directory() / self.directory # First use - format and save tag_name = tag_name or (self.tag.name if self.tag else None) destination = self.format_destination(tag_name) - self.directory = get_relative_to_media_directory(destination) + self.directory = destination # Store absolute path; MediaPathType handles conversion session.commit() # Trigger config save for persistence diff --git a/wrolpi/collections/reorganize.py b/wrolpi/collections/reorganize.py new file mode 100644 index 00000000..1b6ebca5 --- /dev/null +++ b/wrolpi/collections/reorganize.py @@ -0,0 +1,801 @@ +""" +Collection Reorganization Module + +Provides functionality to reorganize files within Collections (Domains/Channels) +when the file_name_format config changes. + +Example: User changes archive format from '%(title)s.%(ext)s' to '%(download_year)s/%(title)s.%(ext)s' +- Before: 'My Article.html' +- After: '2025/My Article.html' +""" +import pathlib +from dataclasses import dataclass +from typing import List, Tuple, Optional + +from sqlalchemy import func +from sqlalchemy.orm import Session, joinedload + +from wrolpi.common import logger, get_media_directory +from wrolpi.db import get_db_session +from wrolpi.files.lib import glob_shared_stem +from .errors import UnknownCollection +from .models import Collection + +logger = logger.getChild(__name__) + +__all__ = [ + 'ReorganizationPreview', + 'get_reorganization_preview', + 'execute_reorganization', + 'get_collections_needing_reorganization', + 'execute_batch_reorganization', + 'get_batch_reorganization_status', +] + + +@dataclass +class ReorganizationPreview: + """Preview of a collection reorganization operation. + + Note: For performance reasons, files_needing_move equals total_files. + We assume all files need moving since computing actual moves for every + file is expensive and would defeat the purpose of this preview. + """ + collection_id: int + collection_name: str + total_files: int # Total FileGroups in collection + files_needing_move: int # Equals total_files (assumed all need moving) + sample_moves: List[dict] # Sample of moves: [{old_path, new_path}, ...] + new_file_format: str # The format that will be applied + current_file_format: Optional[str] # The format currently stored on collection + + +def _get_format_required_variables(file_format: str) -> set: + """Parse a format string and return the set of variable names it uses. + + Example: '%(upload_year)s/%(title)s.%(ext)s' -> {'upload_year', 'title', 'ext'} + """ + import re + # Match %(variable_name)s patterns + return set(re.findall(r'%\(([^)]+)\)s', file_format)) + + +def _video_has_required_metadata(video, file_format: str) -> bool: + """Check if video has the metadata required by the format string. + + Returns False only if the format requires a variable that the video cannot provide. + """ + required = _get_format_required_variables(file_format) + + file_group = video.file_group + info_json = video.get_info_json() or {} + + # Check each potentially-missing variable + # title and ext are always available (have fallbacks) + has_upload_date = bool( + file_group.published_datetime or info_json.get('upload_date') + ) + has_source_id = bool(video.source_id or info_json.get('id')) + has_uploader = bool(info_json.get('uploader') or info_json.get('channel')) + + # Map format variables to their metadata availability + # Variables that depend on upload_date + date_vars = {'upload_year', 'upload_month', 'upload_date', 'upload_date>%Y', 'upload_date>%m'} + # Variables that depend on source_id + id_vars = {'id'} + # Variables that depend on uploader + uploader_vars = {'uploader', 'channel'} + + for var in required: + if var in date_vars and not has_upload_date: + return False + if var in id_vars and not has_source_id: + return False + if var in uploader_vars and not has_uploader: + return False + + return True + + +def _compute_new_path_for_video( + video, + collection_directory: pathlib.Path, + file_format: str = None, +) -> Optional[pathlib.Path]: + """Compute the new path for a video based on current config. + + Args: + video: Video model instance + collection_directory: Base directory for the collection + file_format: Optional format string override, defaults to config value + + Returns None if the video lacks required metadata for the configured format. + """ + from modules.videos.lib import format_video_filename, get_videos_downloader_config + + if file_format is None: + config = get_videos_downloader_config() + file_format = config.file_name_format + + if not _video_has_required_metadata(video, file_format): + logger.info(f'Skipping video {video.id}: missing metadata required by format "{file_format}"') + return None + try: + new_filename = format_video_filename(video, file_format) + return collection_directory / new_filename + except Exception as e: + logger.warning(f'Failed to compute new path for video {video.id}: {e}') + return None + + +def _archive_has_required_metadata(archive, file_format: str) -> bool: + """Check if archive has the metadata required by the format string. + + Returns False only if the format requires a variable that the archive cannot provide. + Archives have robust fallbacks for most variables (dates fallback to now(), + title to 'untitled'). Only domain can be empty when URL is missing. + """ + required = _get_format_required_variables(file_format) + + file_group = archive.file_group + + # Domain requires a URL + has_domain = bool(file_group.url) + + # Domain-related variables + domain_vars = {'domain'} + + for var in required: + if var in domain_vars and not has_domain: + return False + + # All other variables (title, download_*, ext) always have fallbacks + return True + + +def _compute_new_path_for_archive( + archive, + collection_directory: pathlib.Path, + file_format: str = None, +) -> Optional[pathlib.Path]: + """Compute the new path for an archive based on current config. + + Args: + archive: Archive model instance + collection_directory: Base directory for the collection + file_format: Optional format string override, defaults to config value + + Returns None if the archive lacks required metadata for the configured format. + """ + from modules.archive.lib import format_archive_filename_from_archive, get_archive_downloader_config + + if file_format is None: + config = get_archive_downloader_config() + file_format = config.file_name_format + + if not _archive_has_required_metadata(archive, file_format): + logger.info(f'Skipping archive {archive.id}: missing metadata required by format "{file_format}"') + return None + + try: + new_filename = format_archive_filename_from_archive(archive, file_format) + return collection_directory / new_filename + except Exception as e: + logger.warning(f'Failed to compute new path for archive {archive.id}: {e}') + return None + + +def get_reorganization_preview( + collection_id: int, + session: Session = None, + sample_size: int = 10, + exact_count: bool = False, +) -> ReorganizationPreview: + """Get preview with sample of files to be renamed. + + Does NOT compute all moves by default - only samples for preview display. + + Args: + collection_id: The collection to preview reorganization for + session: Database session (optional, will create one if not provided) + sample_size: Number of sample moves to include in preview + exact_count: If True, compute the actual count of files needing moves + (more expensive but accurate for individual collection preview) + + Returns: + ReorganizationPreview with counts and sample moves + + Raises: + UnknownCollection: If collection not found + """ + if session is None: + with get_db_session() as session: + return get_reorganization_preview(collection_id, session, sample_size, exact_count) + + collection = session.query(Collection).filter_by(id=collection_id).one_or_none() + if not collection: + raise UnknownCollection(f"Collection with ID {collection_id} not found") + + if not collection.directory: + raise ValueError(f"Collection '{collection.name}' has no directory. Cannot reorganize.") + + collection_directory = pathlib.Path(collection.directory) + if not collection_directory.is_absolute(): + collection_directory = get_media_directory() / collection_directory + + # Get the current file format from config + current_config_format = collection._get_current_file_format() + + sample_moves = [] + files_needing_move = 0 + total_files = 0 + + if collection.kind == 'channel': + from modules.videos.models import Video, Channel + + # Get the channel for this collection + channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none() + if not channel: + return ReorganizationPreview( + collection_id=collection_id, + collection_name=collection.name, + total_files=0, + files_needing_move=0, + sample_moves=[], + new_file_format=current_config_format or '', + current_file_format=collection.file_format, + ) + + # Use COUNT for total_files (fast) + total_files = session.query(func.count(Video.id)).filter_by(channel_id=channel.id).scalar() + + if exact_count: + # Compute actual count by building full move mappings + move_mappings = _build_move_mappings_for_channel(collection, session) + files_needing_move = len(move_mappings) + else: + # Assume all files need moving (computing actual count is expensive) + files_needing_move = total_files + + # Only load limited videos for sample search + videos = session.query(Video).options( + joinedload(Video.file_group) + ).filter_by(channel_id=channel.id).limit(10).all() + + for video in videos: + if not video.file_group or not video.file_group.primary_path: + continue + + current_path = pathlib.Path(video.file_group.primary_path) + new_path = _compute_new_path_for_video(video, collection_directory) + + if new_path and current_path != new_path: + # Get relative paths for display + media_dir = get_media_directory() + try: + old_relative = str(current_path.relative_to(media_dir)) + except ValueError: + old_relative = str(current_path) + try: + new_relative = str(new_path.relative_to(media_dir)) + except ValueError: + new_relative = str(new_path) + + sample_moves.append({ + 'old_path': old_relative, + 'new_path': new_relative, + }) + if len(sample_moves) >= sample_size: + break + + elif collection.kind == 'domain': + from modules.archive.models import Archive + + # Use COUNT for total_files (fast) + total_files = session.query(func.count(Archive.id)).filter_by(collection_id=collection_id).scalar() + + if exact_count: + # Compute actual count by building full move mappings + move_mappings = _build_move_mappings_for_domain(collection, session) + files_needing_move = len(move_mappings) + else: + # Assume all files need moving (computing actual count is expensive) + files_needing_move = total_files + + # Only load limited archives for sample search + archives = session.query(Archive).options( + joinedload(Archive.file_group) + ).filter_by(collection_id=collection_id).limit(10).all() + + for archive in archives: + if not archive.file_group or not archive.file_group.primary_path: + continue + + current_path = pathlib.Path(archive.file_group.primary_path) + new_path = _compute_new_path_for_archive(archive, collection_directory) + + if new_path and current_path != new_path: + # Get relative paths for display + media_dir = get_media_directory() + try: + old_relative = str(current_path.relative_to(media_dir)) + except ValueError: + old_relative = str(current_path) + try: + new_relative = str(new_path.relative_to(media_dir)) + except ValueError: + new_relative = str(new_path) + + sample_moves.append({ + 'old_path': old_relative, + 'new_path': new_relative, + }) + if len(sample_moves) >= sample_size: + break + + return ReorganizationPreview( + collection_id=collection_id, + collection_name=collection.name, + total_files=total_files, + files_needing_move=files_needing_move, + sample_moves=sample_moves, + new_file_format=current_config_format or '', + current_file_format=collection.file_format, + ) + + +def _build_move_mappings_for_channel( + collection: Collection, + session: Session, +) -> List[Tuple[pathlib.Path, pathlib.Path]]: + """Build move mappings for a channel collection.""" + from modules.videos.models import Video, Channel + + collection_directory = pathlib.Path(collection.directory) + if not collection_directory.is_absolute(): + collection_directory = get_media_directory() / collection_directory + + channel = session.query(Channel).filter_by(collection_id=collection.id).one_or_none() + if not channel: + return [] + + move_mappings = [] + videos = session.query(Video).options( + joinedload(Video.file_group) + ).filter_by(channel_id=channel.id).all() + + for video in videos: + if not video.file_group or not video.file_group.primary_path: + continue + + current_path = pathlib.Path(video.file_group.primary_path) + new_path = _compute_new_path_for_video(video, collection_directory) + + if new_path and current_path != new_path: + move_mappings.append((current_path, new_path)) + + return move_mappings + + +def _build_move_mappings_for_domain( + collection: Collection, + session: Session, +) -> List[Tuple[pathlib.Path, pathlib.Path]]: + """Build move mappings for a domain collection.""" + from modules.archive.models import Archive + + collection_directory = pathlib.Path(collection.directory) + if not collection_directory.is_absolute(): + collection_directory = get_media_directory() / collection_directory + + move_mappings = [] + archives = session.query(Archive).options( + joinedload(Archive.file_group) + ).filter_by(collection_id=collection.id).all() + + for archive in archives: + if not archive.file_group or not archive.file_group.primary_path: + continue + + current_path = pathlib.Path(archive.file_group.primary_path) + new_path = _compute_new_path_for_archive(archive, collection_directory) + + if new_path and current_path != new_path: + move_mappings.append((current_path, new_path)) + + return move_mappings + + +def _check_for_destination_conflicts( + move_mappings: List[Tuple[pathlib.Path, pathlib.Path]], + collection_name: str, +) -> None: + """Check for duplicate destination paths in move mappings. + + Raises: + ReorganizationConflict: If two or more files would move to the same destination + """ + from wrolpi.collections.errors import ReorganizationConflict + + dest_to_sources = {} + for source, dest in move_mappings: + if dest in dest_to_sources: + dest_to_sources[dest].append(source) + else: + dest_to_sources[dest] = [source] + + conflicts = {dest: sources for dest, sources in dest_to_sources.items() if len(sources) > 1} + + if conflicts: + conflict_details = [] + for dest, sources in conflicts.items(): + source_names = [str(s.name) for s in sources] + conflict_details.append(f" {dest.name} <- {', '.join(source_names)}") + + raise ReorganizationConflict( + f"Reorganization of '{collection_name}' would cause {len(conflicts)} filename conflict(s):\n" + + "\n".join(conflict_details) + ) + + +def execute_reorganization( + collection_id: int, + session: Session = None, +) -> str: + """Execute reorganization, returns job_id for tracking. + + Iterates ALL FileGroups and queues moves to FileWorker. + + Args: + collection_id: The collection to reorganize + session: Database session (optional) + + Returns: + job_id for tracking progress via FileWorker status + + Raises: + UnknownCollection: If collection not found + """ + from wrolpi.files.worker import file_worker + + if session is None: + with get_db_session(commit=True) as session: + return execute_reorganization(collection_id, session) + + collection = session.query(Collection).filter_by(id=collection_id).one_or_none() + if not collection: + raise UnknownCollection(f"Collection with ID {collection_id} not found") + + if not collection.directory: + raise ValueError(f"Collection '{collection.name}' has no directory. Cannot reorganize.") + + # Build move mappings based on collection kind + if collection.kind == 'channel': + move_mappings = _build_move_mappings_for_channel(collection, session) + elif collection.kind == 'domain': + move_mappings = _build_move_mappings_for_domain(collection, session) + else: + raise ValueError(f"Unsupported collection kind: {collection.kind}") + + # Check for filename conflicts before proceeding + _check_for_destination_conflicts(move_mappings, collection.name) + + if not move_mappings: + logger.info(f'No files need reorganization for collection {collection.name}') + # Still update the file_format to current config + current_format = collection._get_current_file_format() + if current_format: + collection.file_format = current_format + session.commit() + + # Trigger config save + if collection.kind == 'domain': + from modules.archive.lib import save_domains_config + save_domains_config.activate_switch() + elif collection.kind == 'channel': + from modules.videos.lib import save_channels_config + save_channels_config.activate_switch() + + return '' + + logger.info(f'Queueing reorganization for collection {collection.name}: {len(move_mappings)} files') + + # Get the current file format to update AFTER successful completion + # (deferred update enables retry on failure - file_format only updates on success) + pending_file_format = collection._get_current_file_format() + + # Queue the reorganization with FileWorker + # The pending_file_format will be set by handle_reorganize after successful completion + job_id = file_worker.queue_reorganize( + move_mappings, + collection_id=collection_id, + collection_kind=collection.kind, + pending_file_format=pending_file_format, + ) + + return job_id + + +def get_reorganization_status(job_id: str) -> dict: + """Get status of an in-flight reorganization. + + Args: + job_id: The job ID returned by execute_reorganization + + Returns: + Dict with status information: + - status: 'pending' | 'running' | 'complete' | 'failed' + - total: Total files to move + - completed: Files moved so far + - percent: Completion percentage + - error: Error message if failed + """ + from wrolpi.files.worker import file_worker + + try: + job_status = file_worker.get_job_status(job_id) + except RuntimeError: + return {'status': 'unknown', 'error': f'Job {job_id} not found'} + + # Get current worker status if this job is running + worker_status = file_worker.status + + if job_status == 'complete': + return { + 'status': 'complete', + 'total': worker_status.get('operation_total', 0), + 'completed': worker_status.get('operation_total', 0), + 'percent': 100, + 'error': None, + } + elif job_status == 'pending': + return { + 'status': 'pending', + 'total': 0, + 'completed': 0, + 'percent': 0, + 'error': None, + } + else: + # Job is running + return { + 'status': worker_status.get('status', 'running'), + 'total': worker_status.get('operation_total', 0), + 'completed': worker_status.get('operation_processed', 0), + 'percent': worker_status.get('operation_percent', 0), + 'error': worker_status.get('error'), + } + + +# ============================================================================ +# Batch Reorganization Functions +# ============================================================================ + + +def get_collections_needing_reorganization( + kind: str, + session: Session = None, + sample_size: int = 1, +) -> dict: + """Get all collections of a kind that need reorganization with sample previews. + + Args: + kind: 'channel' or 'domain' + session: Database session (optional) + sample_size: Number of sample moves per collection (default 1) + + Returns: + Dict with: + - collections: List of collection info dicts + - total_collections: Total count needing reorganization + - total_files_needing_move: Sum of all files needing reorganization + - new_file_format: The config format that will be applied + """ + if session is None: + with get_db_session() as session: + return get_collections_needing_reorganization(kind, session, sample_size) + + # Query all collections of this kind with directories + collections = session.query(Collection).filter( + Collection.kind == kind, + Collection.directory.isnot(None), + ).all() + + # Filter to only those needing reorganization + collections_needing_reorg = [c for c in collections if c.needs_reorganization] + + # Get current config format + current_format = '' + if kind == 'channel': + from modules.videos.lib import get_videos_downloader_config + current_format = get_videos_downloader_config().file_name_format + elif kind == 'domain': + from modules.archive.lib import get_archive_downloader_config + current_format = get_archive_downloader_config().file_name_format + + result_collections = [] + total_files_needing_move = 0 + + for collection in collections_needing_reorg: + # Get preview for each collection + try: + preview = get_reorganization_preview(collection.id, session, sample_size=sample_size) + sample_move = preview.sample_moves[0] if preview.sample_moves else None + result_collections.append({ + 'collection_id': collection.id, + 'collection_name': collection.name, + 'total_files': preview.total_files, + 'files_needing_move': preview.files_needing_move, + 'sample_move': sample_move, + }) + total_files_needing_move += preview.files_needing_move + except Exception as e: + logger.warning(f'Failed to get preview for collection {collection.id}: {e}') + result_collections.append({ + 'collection_id': collection.id, + 'collection_name': collection.name, + 'total_files': 0, + 'files_needing_move': 0, + 'sample_move': None, + }) + + return { + 'collections': result_collections, + 'total_collections': len(result_collections), + 'total_files_needing_move': total_files_needing_move, + 'new_file_format': current_format, + } + + +def execute_batch_reorganization(kind: str, session: Session = None) -> dict: + """Execute batch reorganization for all collections of a kind that need it. + + This queues a batch reorganization job that processes collections sequentially. + Each collection is reorganized one at a time to avoid system overload. + If any collection fails, the batch stops and reports which collection failed. + + Args: + kind: 'channel' or 'domain' + session: Database session (optional) + + Returns: + Dict with: + - batch_job_id: Job ID for tracking progress + - message: Status message + - collection_count: Number of collections to process + """ + from wrolpi.files.worker import file_worker + + if session is None: + with get_db_session() as session: + return execute_batch_reorganization(kind, session) + + # Get collections needing reorganization + reorg_info = get_collections_needing_reorganization(kind, session, sample_size=0) + collections_to_process = reorg_info['collections'] + + if not collections_to_process: + logger.info(f'No {kind} collections need reorganization') + return { + 'batch_job_id': '', + 'message': f'No {kind} collections need reorganization', + 'collection_count': 0, + } + + # Extract collection IDs for the batch job + collection_ids = [c['collection_id'] for c in collections_to_process] + + logger.info(f'Queueing batch reorganization for {len(collection_ids)} {kind} collections') + + # Queue the batch reorganization + batch_job_id = file_worker.queue_batch_reorganize(collection_ids, kind) + + return { + 'batch_job_id': batch_job_id, + 'message': 'Batch reorganization started', + 'collection_count': len(collection_ids), + } + + +def get_batch_reorganization_status(batch_job_id: str) -> dict: + """Get status of a batch reorganization job. + + Args: + batch_job_id: The batch job ID returned by execute_batch_reorganization + + Returns: + Dict with: + - status: 'pending' | 'running' | 'complete' | 'failed' | 'unknown' + - total_collections: Total collections in batch + - completed_collections: Number completed so far + - current_collection: Info about currently processing collection (or None) + - overall_percent: Overall completion percentage + - completed: List of completed collection info + - failed_collection: Info about failed collection (or None) + - error: Error message if failed + """ + from wrolpi.files.worker import file_worker + + try: + job_status = file_worker.get_job_status(batch_job_id) + except RuntimeError: + return { + 'status': 'unknown', + 'error': f'Batch job {batch_job_id} not found', + 'total_collections': 0, + 'completed_collections': 0, + 'current_collection': None, + 'overall_percent': 0, + 'completed': [], + 'failed_collection': None, + } + + # Get batch-specific status from worker + worker_status = file_worker.status + batch_status = worker_status.get('batch_status', {}) + + if job_status == 'complete': + completed_list = batch_status.get('completed', []) + return { + 'status': 'complete', + 'total_collections': batch_status.get('total_collections', 0), + 'completed_collections': len(completed_list), + 'current_collection': None, + 'overall_percent': 100, + 'completed': completed_list, + 'failed_collection': None, + 'error': None, + } + elif job_status == 'pending': + return { + 'status': 'pending', + 'total_collections': batch_status.get('total_collections', 0), + 'completed_collections': 0, + 'current_collection': None, + 'overall_percent': 0, + 'completed': [], + 'failed_collection': None, + 'error': None, + } + elif job_status == 'failed': + completed_list = batch_status.get('completed', []) + return { + 'status': 'failed', + 'total_collections': batch_status.get('total_collections', 0), + 'completed_collections': len(completed_list), + 'current_collection': None, + 'overall_percent': batch_status.get('overall_percent', 0), + 'completed': completed_list, + 'failed_collection': batch_status.get('failed_collection'), + 'error': batch_status.get('error'), + } + else: + # Job is running + completed_list = batch_status.get('completed', []) + current = batch_status.get('current_collection') + total = batch_status.get('total_collections', 1) + + # Get fresh file-level progress from worker status (not from batch_status which may be stale) + current_percent = worker_status.get('operation_percent', 0) + + # Update current collection dict with fresh values if it exists + if current: + current = dict(current) # Copy to avoid mutating the original + current['percent'] = current_percent + current['completed'] = worker_status.get('operation_processed', 0) + current['total'] = worker_status.get('operation_total', 0) + + # Calculate overall percent: (completed + current_progress) / total + completed_count = len(completed_list) + overall_percent = int(((completed_count + current_percent / 100) / total) * 100) if total > 0 else 0 + + return { + 'status': 'running', + 'total_collections': total, + 'completed_collections': completed_count, + 'current_collection': current, + 'overall_percent': overall_percent, + 'completed': completed_list, + 'failed_collection': None, + 'error': None, + } diff --git a/wrolpi/collections/schema.py b/wrolpi/collections/schema.py index 5114b176..56c3f10d 100644 --- a/wrolpi/collections/schema.py +++ b/wrolpi/collections/schema.py @@ -52,3 +52,88 @@ class CollectionTagInfoResponse: suggested_directory: str conflict: bool conflict_message: Optional[str] = None + + +@dataclass +class ReorganizationPreviewResponse: + """Response for reorganization preview.""" + collection_id: int + collection_name: str + total_files: int + files_needing_move: int + sample_moves: List[dict] + new_file_format: str + current_file_format: Optional[str] = None + + +@dataclass +class ReorganizationExecuteResponse: + """Response for reorganization execution.""" + job_id: str + message: str + + +@dataclass +class ReorganizationStatusResponse: + """Response for reorganization status.""" + status: str + total: int + completed: int + percent: int + error: Optional[str] = None + + +# ============================================================================ +# Batch Reorganization Schemas +# ============================================================================ + + +@dataclass +class BatchCollectionInfo: + """Info about a single collection in batch reorganization.""" + collection_id: int + collection_name: str + total_files: int + files_needing_move: int + sample_move: Optional[dict] = None + + +@dataclass +class BatchReorganizationListResponse: + """Response for listing collections needing batch reorganization.""" + collections: List[dict] + total_collections: int + total_files_needing_move: int + new_file_format: str + + +@dataclass +class BatchReorganizationExecuteResponse: + """Response for executing batch reorganization.""" + batch_job_id: str + message: str + collection_count: int + + +@dataclass +class BatchCollectionProgress: + """Progress info for a single collection in batch reorganization.""" + id: int + name: str + status: str + total: int = 0 + completed: int = 0 + percent: int = 0 + + +@dataclass +class BatchReorganizationStatusResponse: + """Response for batch reorganization status.""" + status: str + total_collections: int + completed_collections: int + current_collection: Optional[dict] + overall_percent: int + completed: List[dict] + failed_collection: Optional[dict] + error: Optional[str] = None diff --git a/wrolpi/collections/test/test_reorganize.py b/wrolpi/collections/test/test_reorganize.py new file mode 100644 index 00000000..dce65969 --- /dev/null +++ b/wrolpi/collections/test/test_reorganize.py @@ -0,0 +1,1978 @@ +""" +Tests for the collection reorganization feature. +""" +import pytest + +from modules.archive.lib import get_archive_downloader_config +from modules.archive.models import Archive +from modules.videos.lib import get_videos_downloader_config +from modules.videos.models import Video, Channel +from wrolpi.collections import Collection +from wrolpi.collections.errors import UnknownCollection, ReorganizationConflict +from wrolpi.collections.reorganize import ( + get_reorganization_preview, + execute_reorganization, + get_reorganization_status, + get_collections_needing_reorganization, + execute_batch_reorganization, + get_batch_reorganization_status, + ReorganizationPreview, +) +from wrolpi.common import get_media_directory +from wrolpi.db import get_db_session +from wrolpi.files.models import FileGroup + + +def test_collection_file_format_serialization(test_session, test_directory): + """Collection file_format should be persisted, serialized to config, and included in JSON.""" + collection = Collection( + name='test.com', + kind='domain', + directory=test_directory / 'archive' / 'test.com', + file_format='%(download_year)s/%(title)s.%(ext)s' + ) + test_session.add(collection) + test_session.commit() + + # Test column persistence + test_session.refresh(collection) + assert collection.file_format == '%(download_year)s/%(title)s.%(ext)s' + + # Test to_config + config = collection.to_config() + assert config['file_format'] == '%(download_year)s/%(title)s.%(ext)s' + + # Test __json__ + json_data = collection.__json__() + assert 'needs_reorganization' in json_data + assert 'file_format' in json_data + + +def test_collection_needs_reorganization_property(test_session, test_directory): + """Collection.needs_reorganization should be True when file_format differs or is None.""" + collection = Collection( + name='test.com', + kind='domain', + directory=test_directory / 'archive' / 'test.com', + file_format=None # Test None case + ) + test_session.add(collection) + test_session.commit() + + # None format should need reorganization + assert collection.needs_reorganization is True + + # Format matching config should NOT need reorganization + config = get_archive_downloader_config() + collection.file_format = config.file_name_format + test_session.commit() + assert collection.needs_reorganization is False + + # Different format SHOULD need reorganization + collection.file_format = '%(download_year)s/%(title)s.%(ext)s' + test_session.commit() + assert collection.needs_reorganization is True + + +def test_collection_from_config_loads_file_format(test_session, test_directory): + """Collection.from_config should load file_format from config.""" + config_data = { + 'name': 'test.com', + 'kind': 'domain', + 'directory': str(test_directory / 'archive' / 'test.com'), + 'file_format': '%(download_year)s/%(title)s.%(ext)s' + } + + collection = Collection.from_config(test_session, config_data) + test_session.commit() + + assert collection.file_format == '%(download_year)s/%(title)s.%(ext)s' + + +@pytest.mark.parametrize("func_name,error_type,error_match,setup", [ + ("get_reorganization_preview", UnknownCollection, None, "unknown_id"), + ("execute_reorganization", UnknownCollection, None, "unknown_id"), + ("get_reorganization_preview", ValueError, "has no directory", "no_directory"), + ("execute_reorganization", ValueError, "has no directory", "no_directory"), +]) +def test_reorganization_error_handling(func_name, error_type, error_match, setup, test_session, test_directory): + """Both preview and execute should raise appropriate errors for invalid inputs.""" + func = get_reorganization_preview if func_name == "get_reorganization_preview" else execute_reorganization + + if setup == "unknown_id": + # Test with non-existent collection ID + with pytest.raises(error_type): + func(99999, test_session) + elif setup == "no_directory": + # Test with collection that has no directory + collection = Collection( + name='test.com', + kind='domain', + directory=None, + ) + test_session.add(collection) + test_session.commit() + + with pytest.raises(error_type, match=error_match): + func(collection.id, test_session) + + +def test_get_reorganization_preview_domain_collection(test_session, test_directory, archive_factory): + """get_reorganization_preview should work for domain collections.""" + # Create domain collection + domain_dir = test_directory / 'archive' / 'test.com' + domain_dir.mkdir(parents=True, exist_ok=True) + + collection = Collection( + name='test.com', + kind='domain', + directory=domain_dir, + file_format='%(download_datetime)s_%(title)s.%(ext)s' + ) + test_session.add(collection) + test_session.commit() + + # Create an archive using the factory (it uses test_session internally) + archive = archive_factory(domain='test.com', url='https://test.com/article') + # The factory creates a new collection, so associate archive with our collection + archive.collection_id = collection.id + test_session.commit() + + preview = get_reorganization_preview(collection.id, test_session) + + assert isinstance(preview, ReorganizationPreview) + assert preview.collection_id == collection.id + assert preview.collection_name == 'test.com' + assert preview.total_files >= 0 + assert preview.new_file_format is not None + + +@pytest.mark.asyncio +async def test_execute_reorganization_no_files_to_move(async_client, test_directory): + """execute_reorganization should return empty job_id when no files need moving.""" + # Create domain collection with matching file_format + domain_dir = test_directory / 'archive' / 'test.com' + domain_dir.mkdir(parents=True, exist_ok=True) + + config = get_archive_downloader_config() + + with get_db_session(commit=True) as session: + collection = Collection( + name='test.com', + kind='domain', + directory=domain_dir, + file_format=config.file_name_format # Same as config, no reorganization needed + ) + session.add(collection) + session.commit() + collection_id = collection.id + + with get_db_session(commit=True) as session: + job_id = execute_reorganization(collection_id, session) + assert job_id == '' + + +@pytest.mark.asyncio +async def test_get_reorganization_status_unknown_job(async_client): + """get_reorganization_status should return unknown status for non-existent job.""" + status = get_reorganization_status('nonexistent-job-id') + assert status['status'] == 'unknown' + assert 'error' in status + + +def test_filegroup_primary_path_lookup_consistency(test_session, test_directory): + """FileGroup.primary_path returns a Path object, but IN queries use strings. + + This test ensures that the handle_reorganize function correctly converts + the primary_path to a string when building the lookup dictionary. + The bug was: fg_by_path[fg.primary_path] stored Path keys, but lookups + used str(source_path), causing all lookups to fail. + """ + import pathlib + from wrolpi.files.models import FileGroup + + # Create a test file + video_dir = test_directory / 'videos' / 'TestChannel' + video_dir.mkdir(parents=True, exist_ok=True) + video_file = video_dir / 'test_video.mp4' + video_file.touch() + + # Create FileGroup + fg = FileGroup() + fg.directory = video_dir + fg.primary_path = video_file + fg.files = [{'path': 'test_video.mp4', 'mimetype': 'video/mp4'}] + test_session.add(fg) + test_session.commit() + + # Verify primary_path is a Path object (this is how MediaPathType works) + test_session.refresh(fg) + assert isinstance(fg.primary_path, pathlib.Path), \ + f"Expected Path, got {type(fg.primary_path)}" + + # Test the pattern used in handle_reorganize: + # Query returns FileGroups, build lookup with string keys for string lookups + source_paths = [str(video_file)] + file_groups = test_session.query(FileGroup).filter( + FileGroup.primary_path.in_(source_paths) + ).all() + assert len(file_groups) == 1, "Query should find the FileGroup" + + # Build lookup dict - this is where the bug was + # WRONG: fg_by_path[fg.primary_path] (Path key, str lookup fails) + # CORRECT: fg_by_path[str(fg.primary_path)] (str key, str lookup works) + fg_by_path_wrong = {fg.primary_path: fg for fg in file_groups} + fg_by_path_correct = {str(fg.primary_path): fg for fg in file_groups} + + # The source path as string (as used in move_mappings) + lookup_key = str(video_file) + + # This was the bug - Path key doesn't match string lookup + assert lookup_key not in fg_by_path_wrong, \ + "String key should NOT be found in Path-keyed dict" + + # This is the correct behavior after fix + assert lookup_key in fg_by_path_correct, \ + "String key SHOULD be found in string-keyed dict" + + +@pytest.mark.asyncio +async def test_reorganize_updates_filegroup_data_fields(async_client, test_directory, archive_factory): + """FileGroup.data fields should be updated after reorganization. + + After reorganization: + - files[].path is updated to new filename + - data.singlefile_path should match files[].path + - data.screenshot_path should match the new screenshot filename + """ + from wrolpi.files.worker import file_worker, FileTask, FileTaskType + import asyncio + + # Create domain collection with a specific directory + domain_dir = test_directory / 'archive' / 'test.com' + domain_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='test.com', + kind='domain', + directory=domain_dir, + # Old format - files are currently named with this pattern + file_format='%(download_datetime)s_%(title)s.%(ext)s' + ) + session.add(collection) + session.commit() + collection_id = collection.id + + # Create an archive using factory - this creates files with the default format + archive = archive_factory(domain='test.com', url='https://test.com/article', title='Test Article') + + with get_db_session(commit=True) as session: + # Link archive to our collection + archive_obj = session.query(Archive).get(archive.id) + archive_obj.collection_id = collection_id + session.commit() + + # Get the current file paths + fg = archive_obj.file_group + old_primary_path = fg.primary_path + old_singlefile_name = fg.data.get('singlefile_path') + old_screenshot_name = fg.data.get('screenshot_path') + fg_id = fg.id + + # Verify data fields are populated + assert old_singlefile_name is not None, "singlefile_path should be set in data" + assert old_screenshot_name is not None, "screenshot_path should be set in data" + + # Verify data matches files + files_html = [f for f in fg.files if f['path'].endswith('.html') and '.readability' not in f['path']] + assert len(files_html) == 1 + assert files_html[0]['path'] == old_singlefile_name, "data.singlefile_path should match files[].path" + + # Create move mapping: rename files to a new pattern + new_stem = 'renamed_article' + new_primary_path = domain_dir / f'{new_stem}.html' + + move_mappings = [(old_primary_path, new_primary_path)] + + # Execute reorganization via file worker + task = FileTask( + task_type=FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + job_id='test-reorganize-data-bug', + ) + + await file_worker.handle_reorganize(task) + + # Give time for DB updates + await asyncio.sleep(0.1) + + # Verify results + with get_db_session() as session: + fg = session.query(FileGroup).get(fg_id) + + # Verify files were physically renamed + assert new_primary_path.exists(), "New primary file should exist" + assert not old_primary_path.exists(), "Old primary file should not exist" + + # Verify FileGroup.files[] was updated + new_files_html = [f for f in fg.files if f['path'].endswith('.html') and '.readability' not in f['path']] + assert len(new_files_html) == 1 + new_file_name = new_files_html[0]['path'] + assert new_file_name == f'{new_stem}.html', f"files[].path should be updated to new filename, got {new_file_name}" + + # Verify FileGroup.data.singlefile_path was updated to match files[].path + new_singlefile_name = fg.data.get('singlefile_path') + assert new_singlefile_name == new_file_name, \ + f"data.singlefile_path should match files[].path after reorganization. " \ + f"Expected '{new_file_name}', got '{new_singlefile_name}'" + + # Verify FileGroup.data.screenshot_path was updated + new_screenshot_name = fg.data.get('screenshot_path') + expected_screenshot = f'{new_stem}.png' + assert new_screenshot_name == expected_screenshot, \ + f"data.screenshot_path should be updated after reorganization. " \ + f"Expected '{expected_screenshot}', got '{new_screenshot_name}'" + + # Verify readability paths were also updated + new_readability_name = fg.data.get('readability_path') + expected_readability = f'{new_stem}.readability.html' + assert new_readability_name == expected_readability, \ + f"data.readability_path should be updated after reorganization. " \ + f"Expected '{expected_readability}', got '{new_readability_name}'" + + +# ============================================================================ +# Batch Reorganization Tests +# ============================================================================ + + +@pytest.mark.parametrize("kind,config_getter,base_dir,names", [ + ("channel", "get_videos_downloader_config", "videos", ["Channel1", "Channel2", "Channel3"]), + ("domain", "get_archive_downloader_config", "archive", ["example.com", "test.org", "done.net"]), +]) +def test_get_collections_needing_reorganization(test_session, test_directory, kind, config_getter, base_dir, names): + """Create 3 collections, 2 need reorganization, verify only 2 returned.""" + # Get current config format + config = get_videos_downloader_config() if config_getter == "get_videos_downloader_config" else get_archive_downloader_config() + + # Create base directory + parent_dir = test_directory / base_dir + parent_dir.mkdir(parents=True, exist_ok=True) + + # Collection 1: Needs reorganization (different format) + dir1 = parent_dir / names[0] + dir1.mkdir(parents=True, exist_ok=True) + collection1 = Collection( + name=names[0], + kind=kind, + directory=dir1, + file_format='%(old_format)s.%(ext)s' # Different from config + ) + test_session.add(collection1) + test_session.flush() + if kind == 'channel': + test_session.add(Channel(name=names[0], collection_id=collection1.id)) + + # Collection 2: Needs reorganization (None format) + dir2 = parent_dir / names[1] + dir2.mkdir(parents=True, exist_ok=True) + collection2 = Collection( + name=names[1], + kind=kind, + directory=dir2, + file_format=None # Never been synced + ) + test_session.add(collection2) + test_session.flush() + if kind == 'channel': + test_session.add(Channel(name=names[1], collection_id=collection2.id)) + + # Collection 3: Does NOT need reorganization (matches config) + dir3 = parent_dir / names[2] + dir3.mkdir(parents=True, exist_ok=True) + collection3 = Collection( + name=names[2], + kind=kind, + directory=dir3, + file_format=config.file_name_format # Matches config + ) + test_session.add(collection3) + test_session.flush() + if kind == 'channel': + test_session.add(Channel(name=names[2], collection_id=collection3.id)) + + test_session.commit() + + # Get collections needing reorganization + result = get_collections_needing_reorganization(kind, test_session) + + assert 'collections' in result + assert result['total_collections'] == 2 + collection_ids = {c['collection_id'] for c in result['collections']} + assert collection1.id in collection_ids + assert collection2.id in collection_ids + assert collection3.id not in collection_ids + + +def test_get_collections_needing_reorganization_empty(test_session, test_directory): + """When no collections need reorganization, return empty list.""" + config = get_videos_downloader_config() + + # Create channel that does NOT need reorganization + videos_dir = test_directory / 'videos' + channel_dir = videos_dir / 'SyncedChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + collection = Collection( + name='SyncedChannel', + kind='channel', + directory=channel_dir, + file_format=config.file_name_format + ) + test_session.add(collection) + test_session.flush() + channel = Channel(name='SyncedChannel', collection_id=collection.id) + test_session.add(channel) + test_session.commit() + + result = get_collections_needing_reorganization('channel', test_session) + + assert result['collections'] == [] + assert result['total_collections'] == 0 + assert result['total_files_needing_move'] == 0 + + +def test_get_collections_needing_reorganization_includes_sample_move( + test_session, test_directory, video_factory +): + """Each collection should include one sample file move preview.""" + config = get_videos_downloader_config() + + # Create channel with old format + videos_dir = test_directory / 'videos' + channel_dir = videos_dir / 'TestChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + collection = Collection( + name='TestChannel', + kind='channel', + directory=channel_dir, + file_format='%(old_format)s.%(ext)s' + ) + test_session.add(collection) + test_session.flush() + channel = Channel(name='TestChannel', collection_id=collection.id, directory=channel_dir) + test_session.add(channel) + test_session.commit() + + # Create a video in this channel using factory + video = video_factory(channel_id=channel.id) + + result = get_collections_needing_reorganization('channel', test_session, sample_size=1) + + assert len(result['collections']) == 1 + collection_info = result['collections'][0] + assert collection_info['collection_id'] == collection.id + assert collection_info['collection_name'] == 'TestChannel' + assert 'sample_move' in collection_info + # sample_move should be a dict with old_path and new_path (or None if no files need moving) + + +@pytest.mark.asyncio +async def test_batch_status_includes_batch_job_id(async_client, test_directory, video_factory): + """batch_status should include batch_job_id so frontend can resume polling. + + Bug: Frontend sets batchJobId='active' as a marker when resuming, but then + uses this literal string for API calls, causing "Batch job active not found". + + Fix: batch_status must include the real batch_job_id so frontend can use it. + """ + from datetime import datetime, timezone + from wrolpi.files.worker import file_worker + import asyncio + + # Create channel needing reorganization + channel_dir = test_directory / 'videos' / 'TestChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='TestChannel', + kind='channel', + directory=channel_dir, + file_format='%(old_format)s.%(ext)s' # Different from current config + ) + session.add(collection) + session.flush() + channel = Channel(name='TestChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + + # Create video with proper metadata + video = video_factory( + channel_id=1, + title='Test Video', + with_video_file=True, + with_info_json={'uploader': 'Tester', 'id': 'abc123'}, + upload_date=datetime(2024, 6, 15, tzinfo=timezone.utc), + ) + + # Start batch reorganization + with get_db_session() as session: + result = execute_batch_reorganization('channel', session) + + batch_job_id = result['batch_job_id'] + assert batch_job_id, "Should return batch_job_id" + + # Transfer the task from pending queue to worker queue, then start processing + file_worker.transfer_queue() + + # Start the worker processing in a task (it will set up status before processing) + process_task = asyncio.create_task(file_worker.process_queue()) + + # Wait briefly for worker to start processing and set status + await asyncio.sleep(0.2) + + # Check that batch_status includes the batch_job_id + worker_status = file_worker.status + batch_status = worker_status.get('batch_status', {}) + + # Cancel the processing task to clean up + process_task.cancel() + try: + await process_task + except asyncio.CancelledError: + pass + + assert 'batch_job_id' in batch_status, ( + "batch_status must include batch_job_id for frontend resume. " + f"Got: {list(batch_status.keys())}" + ) + assert batch_status['batch_job_id'] == batch_job_id, ( + f"batch_status.batch_job_id should match returned ID. " + f"Expected: {batch_job_id}, got: {batch_status.get('batch_job_id')}" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind,base_dir,names", [ + ("channel", "videos", ["Channel1", "Channel2"]), + ("domain", "archive", ["example.com", "test.org"]), +]) +async def test_execute_batch_reorganization(async_client, test_directory, kind, base_dir, names): + """Execute batch reorganization, verify all collections processed.""" + parent_dir = test_directory / base_dir + parent_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + # Create two collections that need reorganization + dir1 = parent_dir / names[0] + dir1.mkdir(parents=True, exist_ok=True) + collection1 = Collection( + name=names[0], + kind=kind, + directory=dir1, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection1) + session.flush() + if kind == 'channel': + session.add(Channel(name=names[0], collection_id=collection1.id, directory=dir1)) + + dir2 = parent_dir / names[1] + dir2.mkdir(parents=True, exist_ok=True) + collection2 = Collection( + name=names[1], + kind=kind, + directory=dir2, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection2) + session.flush() + if kind == 'channel': + session.add(Channel(name=names[1], collection_id=collection2.id, directory=dir2)) + + session.commit() + + # Execute batch reorganization + with get_db_session() as session: + result = execute_batch_reorganization(kind, session) + + assert 'batch_job_id' in result + assert result['batch_job_id'].startswith('batch-reorganize-') + assert result['collection_count'] >= 0 # May be 0 if no files to move + + +def test_batch_reorganization_status_tracking(test_session): + """Status should show overall + per-collection progress.""" + # Test with a non-existent batch job + status = get_batch_reorganization_status('nonexistent-batch-job') + + assert status['status'] == 'unknown' + assert 'error' in status + + +@pytest.mark.asyncio +async def test_batch_reorganization_no_collections_needing_reorganization(async_client, test_directory): + """Execute batch reorganization when no collections need it.""" + config = get_videos_downloader_config() + + videos_dir = test_directory / 'videos' + channel_dir = videos_dir / 'SyncedChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='SyncedChannel', + kind='channel', + directory=channel_dir, + file_format=config.file_name_format # Already synced + ) + session.add(collection) + session.flush() + channel = Channel(name='SyncedChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + + with get_db_session() as session: + result = execute_batch_reorganization('channel', session) + + # Should return empty batch job ID since nothing to do + assert result['batch_job_id'] == '' + assert result['collection_count'] == 0 + + +# ============================================================================ +# Tag Preservation Tests +# ============================================================================ + + +@pytest.mark.asyncio +async def test_reorganize_preserves_tag_association(async_client, test_directory, video_factory, tag_factory, + test_tags_config): + """FileGroup tags should be preserved after reorganization. + + TagFile uses file_group_id (not paths), so tags should automatically + survive when FileGroup paths are updated during reorganization. + """ + from wrolpi.files.worker import file_worker, FileTask, FileTaskType + import asyncio + + # Create channel collection with a specific directory + channel_dir = test_directory / 'videos' / 'test_channel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='Test Channel', + kind='channel', + directory=channel_dir, + file_format='%(title)s.%(ext)s' + ) + session.add(collection) + session.flush() + channel = Channel(name='Test Channel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + + # Create tag first + tag = await tag_factory('mytag') + + # Create video in the channel + video = video_factory(channel_id=channel_id, title='test_video', with_video_file=True) + + with get_db_session(commit=True) as session: + video_obj = session.query(Video).get(video.id) + video_obj.file_group.add_tag(session, tag.id) + fg_id = video_obj.file_group.id + old_primary_path = video_obj.file_group.primary_path + session.commit() + + # Verify tag was added + with get_db_session() as session: + fg = session.query(FileGroup).get(fg_id) + assert fg.tag_names == ['mytag'], "Tag should be applied before reorganization" + + # Create move mapping: rename files to a new pattern + new_stem = 'renamed_video' + new_primary_path = channel_dir / f'{new_stem}.mp4' + move_mappings = [(old_primary_path, new_primary_path)] + + # Execute reorganization via file worker + task = FileTask( + task_type=FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + job_id='test-reorganize-tags', + ) + await file_worker.handle_reorganize(task) + await asyncio.sleep(0.1) + + # Verify tags preserved after file move + with get_db_session() as session: + fg = session.query(FileGroup).get(fg_id) + assert fg.tag_names == ['mytag'], "Tag should be preserved after reorganization" + assert fg.primary_path == new_primary_path, "File should have moved" + + +@pytest.mark.asyncio +async def test_reorganize_preserves_multiple_tags(async_client, test_directory, video_factory, tag_factory, + test_tags_config): + """FileGroup with multiple tags should retain all tags after reorganization.""" + from wrolpi.files.worker import file_worker, FileTask, FileTaskType + import asyncio + + # Create channel collection + channel_dir = test_directory / 'videos' / 'multi_tag_channel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='Multi Tag Channel', + kind='channel', + directory=channel_dir, + file_format='%(title)s.%(ext)s' + ) + session.add(collection) + session.flush() + channel = Channel(name='Multi Tag Channel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + + # Create multiple tags + tag1 = await tag_factory('alpha') + tag2 = await tag_factory('beta') + tag3 = await tag_factory('gamma') + + # Create video and add all tags + video = video_factory(channel_id=channel_id, title='multi_tagged_video', with_video_file=True) + + with get_db_session(commit=True) as session: + video_obj = session.query(Video).get(video.id) + video_obj.file_group.add_tag(session, tag1.id) + video_obj.file_group.add_tag(session, tag2.id) + video_obj.file_group.add_tag(session, tag3.id) + fg_id = video_obj.file_group.id + old_primary_path = video_obj.file_group.primary_path + session.commit() + + # Verify all tags were added + with get_db_session() as session: + fg = session.query(FileGroup).get(fg_id) + assert sorted(fg.tag_names) == ['alpha', 'beta', 'gamma'], "All tags should be applied" + + # Create move mapping + new_primary_path = channel_dir / 'reorganized_multi_tag.mp4' + move_mappings = [(old_primary_path, new_primary_path)] + + # Execute reorganization + task = FileTask( + task_type=FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + job_id='test-reorganize-multi-tags', + ) + await file_worker.handle_reorganize(task) + await asyncio.sleep(0.1) + + # Verify all tags preserved + with get_db_session() as session: + fg = session.query(FileGroup).get(fg_id) + assert sorted(fg.tag_names) == ['alpha', 'beta', 'gamma'], \ + "All tags should be preserved after reorganization" + assert fg.primary_path == new_primary_path, "File should have moved" + + +@pytest.mark.asyncio +async def test_reorganize_updates_tag_directory_links(async_client, test_directory, video_factory, tag_factory, + test_tags_config, await_switches): + """Tag directory hardlinks should point to new file locations after reorganization. + + After reorganization: + - Old hardlinks should be removed + - New hardlinks should exist at new paths + - Hardlinks should actually link to the moved files + """ + from wrolpi.files.worker import file_worker, FileTask, FileTaskType + from wrolpi.tags import sync_tags_directory, get_tags_directory + from wrolpi.common import is_hardlinked + import asyncio + + # Create channel collection + channel_dir = test_directory / 'videos' / 'tag_link_channel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='Tag Link Channel', + kind='channel', + directory=channel_dir, + file_format='%(title)s.%(ext)s' + ) + session.add(collection) + session.flush() + channel = Channel(name='Tag Link Channel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + + # Create tag and video + tag = await tag_factory('linked_tag') + video = video_factory(channel_id=channel_id, title='link_test_video', with_video_file=True) + + with get_db_session(commit=True) as session: + video_obj = session.query(Video).get(video.id) + video_obj.file_group.add_tag(session, tag.id) + fg_id = video_obj.file_group.id + old_primary_path = video_obj.file_group.primary_path + session.commit() + + # Sync tags directory to create initial hardlinks + sync_tags_directory.activate_switch() + await await_switches() + + # Verify initial hardlinks exist + tags_dir = get_tags_directory() + old_link_path = tags_dir / 'linked_tag' / old_primary_path.name + assert old_link_path.exists(), "Initial tag directory link should exist" + assert is_hardlinked(old_link_path), "Initial tag directory link should be a hardlink" + + # Execute reorganization + new_primary_path = channel_dir / 'renamed_link_test.mp4' + move_mappings = [(old_primary_path, new_primary_path)] + + task = FileTask( + task_type=FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + job_id='test-reorganize-tag-links', + ) + await file_worker.handle_reorganize(task) + await asyncio.sleep(0.1) + + # Manually sync tags directory (this is the potential bug - reorganize may not call this) + sync_tags_directory.activate_switch() + await await_switches() + + # Verify new hardlinks exist at new paths + new_link_path = tags_dir / 'linked_tag' / new_primary_path.name + assert new_link_path.exists(), f"New tag directory link should exist at {new_link_path}" + assert is_hardlinked(new_link_path), "New tag directory link should be a hardlink" + + # Verify old hardlinks were removed + assert not old_link_path.exists(), "Old tag directory link should be removed" + + # Verify the new file exists and is the same inode as the hardlink + assert new_primary_path.exists(), "New primary file should exist" + assert new_primary_path.stat().st_ino == new_link_path.stat().st_ino, \ + "New file and new link should have the same inode" + + +@pytest.mark.asyncio +async def test_batch_reorganize_preserves_tags(async_client, test_directory, video_factory, tag_factory, + test_tags_config): + """Batch reorganization should preserve tags across multiple collections.""" + from wrolpi.files.worker import file_worker + import asyncio + + videos_dir = test_directory / 'videos' + videos_dir.mkdir(parents=True, exist_ok=True) + + # Create two channels with different file formats + with get_db_session(commit=True) as session: + channel1_dir = videos_dir / 'BatchChannel1' + channel1_dir.mkdir(parents=True, exist_ok=True) + collection1 = Collection( + name='BatchChannel1', + kind='channel', + directory=channel1_dir, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection1) + session.flush() + channel1 = Channel(name='BatchChannel1', collection_id=collection1.id, directory=channel1_dir) + session.add(channel1) + + channel2_dir = videos_dir / 'BatchChannel2' + channel2_dir.mkdir(parents=True, exist_ok=True) + collection2 = Collection( + name='BatchChannel2', + kind='channel', + directory=channel2_dir, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection2) + session.flush() + channel2 = Channel(name='BatchChannel2', collection_id=collection2.id, directory=channel2_dir) + session.add(channel2) + + session.commit() + channel1_id = channel1.id + channel2_id = channel2.id + + # Create tags + tag_a = await tag_factory('batch_tag_a') + tag_b = await tag_factory('batch_tag_b') + + # Create videos in each channel with different tags + video1 = video_factory(channel_id=channel1_id, title='batch_video_1', with_video_file=True) + video2 = video_factory(channel_id=channel2_id, title='batch_video_2', with_video_file=True) + + with get_db_session(commit=True) as session: + video1_obj = session.query(Video).get(video1.id) + video1_obj.file_group.add_tag(session, tag_a.id) + fg1_id = video1_obj.file_group.id + + video2_obj = session.query(Video).get(video2.id) + video2_obj.file_group.add_tag(session, tag_b.id) + fg2_id = video2_obj.file_group.id + + session.commit() + + # Verify tags before batch reorganization + with get_db_session() as session: + fg1 = session.query(FileGroup).get(fg1_id) + fg2 = session.query(FileGroup).get(fg2_id) + assert fg1.tag_names == ['batch_tag_a'] + assert fg2.tag_names == ['batch_tag_b'] + + # Execute batch reorganization + with get_db_session() as session: + result = execute_batch_reorganization('channel', session) + + # Give time for batch operations + await asyncio.sleep(0.5) + + # Verify tags preserved after batch reorganization + with get_db_session() as session: + fg1 = session.query(FileGroup).get(fg1_id) + fg2 = session.query(FileGroup).get(fg2_id) + assert fg1.tag_names == ['batch_tag_a'], \ + "Tag on first channel's video should be preserved after batch reorganization" + assert fg2.tag_names == ['batch_tag_b'], \ + "Tag on second channel's video should be preserved after batch reorganization" + + +# ============================================================================ +# Partial Reorganization and Resume Tests +# ============================================================================ + + +@pytest.mark.asyncio +async def test_reorganize_file_format_updated_after_completion(async_client, test_directory, video_factory): + """Verify that file_format is NOT changed until after reorganization completes. + + This is critical for resumability - if file_format is updated before files are moved, + the user cannot retry a failed reorganization because needs_reorganization would return False. + """ + from wrolpi.files.worker import file_worker, FileTask, FileTaskType + import asyncio + + # Create channel collection with old file_format + channel_dir = test_directory / 'videos' / 'DeferredFormatChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + old_format = '%(old_format)s.%(ext)s' + with get_db_session(commit=True) as session: + collection = Collection( + name='DeferredFormatChannel', + kind='channel', + directory=channel_dir, + file_format=old_format + ) + session.add(collection) + session.flush() + channel = Channel(name='DeferredFormatChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + collection_id = collection.id + + # Create video + video = video_factory(channel_id=channel_id, title='format_test_video', with_video_file=True) + + with get_db_session() as session: + video_obj = session.query(Video).get(video.id) + old_primary_path = video_obj.file_group.primary_path + + # Get the current config format (what it should become after reorganization) + config = get_videos_downloader_config() + new_format = config.file_name_format + + # Create move mapping + new_primary_path = channel_dir / 'renamed_format_test.mp4' + move_mappings = [(old_primary_path, new_primary_path)] + + # Execute reorganization directly via handle_reorganize + task = FileTask( + task_type=FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + collection_id=collection_id, + job_id='test-file-format-deferred', + pending_file_format=new_format, # Pass the pending format + ) + + # Before completion, file_format should still be old + with get_db_session() as session: + collection = session.query(Collection).get(collection_id) + assert collection.file_format == old_format, \ + "file_format should NOT be updated before reorganization completes" + + await file_worker.handle_reorganize(task) + await asyncio.sleep(0.1) + + # After completion, file_format should now be updated + with get_db_session() as session: + collection = session.query(Collection).get(collection_id) + assert collection.file_format == new_format, \ + "file_format should be updated after reorganization completes" + assert collection.needs_reorganization is False, \ + "needs_reorganization should be False after successful reorganization" + + +@pytest.mark.asyncio +async def test_reorganize_can_retry_after_partial_failure(async_client, test_directory, video_factory): + """Verify that reorganization can be retried after mid-process failure. + + After partial failure: + - Some files may have been moved + - file_format should NOT have been updated (allowing retry) + - needs_reorganization should still return True + """ + from wrolpi.files.worker import file_worker + import shutil + + # Create channel collection + channel_dir = test_directory / 'videos' / 'PartialFailureChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + old_format = '%(old_format)s.%(ext)s' + with get_db_session(commit=True) as session: + collection = Collection( + name='PartialFailureChannel', + kind='channel', + directory=channel_dir, + file_format=old_format + ) + session.add(collection) + session.flush() + channel = Channel(name='PartialFailureChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + collection_id = collection.id + + # Create multiple videos + video1 = video_factory(channel_id=channel_id, title='partial_video_1', with_video_file=True) + video2 = video_factory(channel_id=channel_id, title='partial_video_2', with_video_file=True) + video3 = video_factory(channel_id=channel_id, title='partial_video_3', with_video_file=True) + + with get_db_session() as session: + video1_obj = session.query(Video).get(video1.id) + video2_obj = session.query(Video).get(video2.id) + video3_obj = session.query(Video).get(video3.id) + + path1 = video1_obj.file_group.primary_path + path2 = video2_obj.file_group.primary_path + path3 = video3_obj.file_group.primary_path + + # Simulate partial failure: move video1 manually (as if reorganization partially completed) + dest1 = channel_dir / 'moved_partial_video_1.mp4' + shutil.move(str(path1), str(dest1)) + + # Verify setup - file moved but DB still points to old location + assert dest1.exists(), "Video 1 should be at destination" + assert not path1.exists(), "Video 1 should not be at source" + + with get_db_session() as session: + video1_obj = session.query(Video).get(video1.id) + assert video1_obj.file_group.primary_path == path1, \ + "DB should still point to old location (simulating partial failure before DB update)" + + # Verify needs_reorganization is still True (file_format not updated) + with get_db_session() as session: + collection = session.query(Collection).get(collection_id) + assert collection.file_format == old_format, \ + "file_format should still be old after partial failure" + assert collection.needs_reorganization is True, \ + "needs_reorganization should still be True after partial failure" + + +@pytest.mark.asyncio +async def test_reorganize_handles_already_moved_files(async_client, test_directory, video_factory): + """Verify graceful handling when files exist at destination but DB points to old location. + + This is a recovery case - files were moved but DB update didn't complete (e.g., power loss). + The reorganization should: + - Detect that source doesn't exist but destination does + - Update DB to match destination paths + - Not error about missing source + """ + from wrolpi.files.worker import file_worker, FileTask, FileTaskType + import asyncio + import shutil + + # Create channel collection + channel_dir = test_directory / 'videos' / 'RecoveryChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='RecoveryChannel', + kind='channel', + directory=channel_dir, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection) + session.flush() + channel = Channel(name='RecoveryChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + collection_id = collection.id + + # Create video + video = video_factory(channel_id=channel_id, title='recovery_video', with_video_file=True) + + with get_db_session() as session: + video_obj = session.query(Video).get(video.id) + old_primary_path = video_obj.file_group.primary_path + fg_id = video_obj.file_group.id + + # Record original file content for verification + original_content = old_primary_path.read_bytes() + + # Manually move video to destination (simulating partial completion) + dest_path = channel_dir / 'recovered_video.mp4' + shutil.move(str(old_primary_path), str(dest_path)) + + # Verify setup + assert not old_primary_path.exists(), "Source should not exist" + assert dest_path.exists(), "Destination should exist (manually moved)" + + with get_db_session() as session: + fg = session.query(FileGroup).get(fg_id) + assert fg.primary_path == old_primary_path, "DB should still point to source" + + # Now run reorganization - it should recover by updating DB + move_mappings = [(old_primary_path, dest_path)] + + config = get_videos_downloader_config() + task = FileTask( + task_type=FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + collection_id=collection_id, + job_id='test-recovery', + pending_file_format=config.file_name_format, + ) + + await file_worker.handle_reorganize(task) + await asyncio.sleep(0.1) + + # Verify recovery: DB should now point to destination + with get_db_session() as session: + fg = session.query(FileGroup).get(fg_id) + assert fg.primary_path == dest_path, \ + f"DB should be updated to destination path after recovery. Got {fg.primary_path}" + assert fg.directory == dest_path.parent, \ + "Directory should be updated to destination directory" + + # Verify file content is intact + assert dest_path.read_bytes() == original_content, "File content should be preserved" + + +@pytest.mark.asyncio +async def test_reorganize_handles_mixed_state(async_client, test_directory, video_factory): + """Verify reorganization handles a mix of filesystem states idempotently. + + - Video 1: Manually move to destination (DB points to source, file at dest) - recovery case + - Video 2: Leave at source (normal case - DB and filesystem match) + - Video 3: Create copy at both source and destination (conflict case) + + Expected results: + - Video 1: DB updated to dest path + - Video 2: File moved to dest, DB updated + - Video 3: Skipped with warning (both exist) + """ + from wrolpi.files.worker import file_worker, FileTask, FileTaskType + import asyncio + import shutil + + # Create channel collection + channel_dir = test_directory / 'videos' / 'MixedStateChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='MixedStateChannel', + kind='channel', + directory=channel_dir, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection) + session.flush() + channel = Channel(name='MixedStateChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + collection_id = collection.id + + # Create 3 videos + video1 = video_factory(channel_id=channel_id, title='mixed_video_1', with_video_file=True) + video2 = video_factory(channel_id=channel_id, title='mixed_video_2', with_video_file=True) + video3 = video_factory(channel_id=channel_id, title='mixed_video_3', with_video_file=True) + + with get_db_session() as session: + v1_obj = session.query(Video).get(video1.id) + v2_obj = session.query(Video).get(video2.id) + v3_obj = session.query(Video).get(video3.id) + + path1 = v1_obj.file_group.primary_path + path2 = v2_obj.file_group.primary_path + path3 = v3_obj.file_group.primary_path + + fg1_id = v1_obj.file_group.id + fg2_id = v2_obj.file_group.id + fg3_id = v3_obj.file_group.id + + dest1 = channel_dir / 'reorganized_mixed_1.mp4' + dest2 = channel_dir / 'reorganized_mixed_2.mp4' + dest3 = channel_dir / 'reorganized_mixed_3.mp4' + + # Setup mixed states + # Video 1: Move to dest (recovery case) + shutil.move(str(path1), str(dest1)) + + # Video 2: Leave at source (normal case) + # (no action needed) + + # Video 3: Copy to both (conflict case) + shutil.copy(str(path3), str(dest3)) + + # Build move mappings + move_mappings = [ + (path1, dest1), + (path2, dest2), + (path3, dest3), + ] + + config = get_videos_downloader_config() + task = FileTask( + task_type=FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + collection_id=collection_id, + job_id='test-mixed-state', + pending_file_format=config.file_name_format, + ) + + await file_worker.handle_reorganize(task) + await asyncio.sleep(0.1) + + with get_db_session() as session: + # Video 1: Should have DB updated to dest (recovery) + fg1 = session.query(FileGroup).get(fg1_id) + assert fg1.primary_path == dest1, \ + f"Video 1 should have DB updated to dest path (recovery). Got {fg1.primary_path}" + + # Video 2: Should be moved and DB updated (normal) + fg2 = session.query(FileGroup).get(fg2_id) + assert fg2.primary_path == dest2, \ + f"Video 2 should be moved to dest. Got {fg2.primary_path}" + assert dest2.exists(), "Video 2 file should exist at destination" + assert not path2.exists(), "Video 2 file should not exist at source" + + # Video 3: Should be skipped (both exist - conflict) + fg3 = session.query(FileGroup).get(fg3_id) + assert fg3.primary_path == path3, \ + f"Video 3 should remain at source (conflict skipped). Got {fg3.primary_path}" + assert path3.exists(), "Video 3 should still exist at source" + assert dest3.exists(), "Video 3 should still exist at dest" + + +@pytest.mark.asyncio +async def test_batch_reorganize_can_resume_after_collection_failure(async_client, test_directory, video_factory): + """Verify batch reorganization can resume after one collection fails. + + - Create multiple channels with videos + - Batch reorganize, simulate failure on second collection + - Verify first collection was completed + - Verify second collection can be retried individually + - Verify file_format on failed collection was NOT updated + """ + # This test verifies that batch failures are recoverable + # The key assertion is that file_format is not updated for failed collections + from wrolpi.files.worker import file_worker + + videos_dir = test_directory / 'videos' + videos_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + # Create two channels that need reorganization + channel1_dir = videos_dir / 'BatchResumeChannel1' + channel1_dir.mkdir(parents=True, exist_ok=True) + collection1 = Collection( + name='BatchResumeChannel1', + kind='channel', + directory=channel1_dir, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection1) + session.flush() + channel1 = Channel(name='BatchResumeChannel1', collection_id=collection1.id, directory=channel1_dir) + session.add(channel1) + collection1_id = collection1.id + channel1_id = channel1.id + + channel2_dir = videos_dir / 'BatchResumeChannel2' + channel2_dir.mkdir(parents=True, exist_ok=True) + collection2 = Collection( + name='BatchResumeChannel2', + kind='channel', + directory=channel2_dir, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection2) + session.flush() + channel2 = Channel(name='BatchResumeChannel2', collection_id=collection2.id, directory=channel2_dir) + session.add(channel2) + collection2_id = collection2.id + + session.commit() + + # Create video in each channel + video1 = video_factory(channel_id=channel1_id, title='batch_resume_video_1', with_video_file=True) + + # Verify both collections need reorganization + with get_db_session() as session: + c1 = session.query(Collection).get(collection1_id) + c2 = session.query(Collection).get(collection2_id) + assert c1.needs_reorganization is True + assert c2.needs_reorganization is True + + # Execute batch reorganization for first collection only + # (simulating partial batch completion) + with get_db_session(commit=True) as session: + job_id = execute_reorganization(collection1_id, session) + if job_id: + # Process the reorganization + file_worker.transfer_queue() + await file_worker.process_queue() + + # Verify first collection completed + with get_db_session() as session: + c1 = session.query(Collection).get(collection1_id) + c2 = session.query(Collection).get(collection2_id) + + # First collection should be done (file_format updated) + # Note: This tests current behavior - may need adjustment based on implementation + config = get_videos_downloader_config() + assert c1.file_format == config.file_name_format, \ + "First collection's file_format should be updated" + assert c1.needs_reorganization is False, \ + "First collection should no longer need reorganization" + + # Second collection should still need reorganization + assert c2.file_format == '%(old_format)s.%(ext)s', \ + "Second collection's file_format should NOT be updated (batch didn't process it)" + assert c2.needs_reorganization is True, \ + "Second collection should still need reorganization" + + +# ============================================================================ +# Filename Conflict Detection Tests +# ============================================================================ + + +@pytest.mark.asyncio +async def test_reorganize_skips_deleted_files_gracefully(async_client, test_directory, video_factory): + """Reorganization should skip move_mappings for missing files, not fail. + + Tests the handle_reorganize behavior when a move_mapping references a file + that no longer exists (neither source nor destination). This tests the + "neither exists" branch in handle_reorganize which should: + - Log a warning + - Skip the file + - Continue processing remaining files + - Complete successfully + + Note: We also delete the Video/FileGroup records to prevent the video_modeler + from creating a new Video and failing during post-processing. + """ + from wrolpi.files.worker import file_worker, FileTask, FileTaskType + import asyncio + import os + + # Create channel collection + channel_dir = test_directory / 'videos' / 'DeletedFilesChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='DeletedFilesChannel', + kind='channel', + directory=channel_dir, + file_format='%(old_format)s.%(ext)s' + ) + session.add(collection) + session.flush() + channel = Channel(name='DeletedFilesChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + collection_id = collection.id + + # Create 3 videos + video1 = video_factory(channel_id=channel_id, title='existing_video_1', with_video_file=True) + video2 = video_factory(channel_id=channel_id, title='deleted_video', with_video_file=True) + video3 = video_factory(channel_id=channel_id, title='existing_video_3', with_video_file=True) + + with get_db_session() as session: + v1_obj = session.query(Video).get(video1.id) + v2_obj = session.query(Video).get(video2.id) + v3_obj = session.query(Video).get(video3.id) + + path1 = v1_obj.file_group.primary_path + path2 = v2_obj.file_group.primary_path + path3 = v3_obj.file_group.primary_path + + fg1_id = v1_obj.file_group.id + fg2_id = v2_obj.file_group.id + fg3_id = v3_obj.file_group.id + + # Simulate user deleting video 2's file from filesystem + # Also delete the Video and FileGroup records to prevent post-processing failure + # (The video_modeler would otherwise create a new Video and try to ffprobe it) + os.remove(str(path2)) + with get_db_session(commit=True) as session: + video_to_delete = session.query(Video).get(video2.id) + fg_to_delete = session.query(FileGroup).get(fg2_id) + session.delete(video_to_delete) + session.delete(fg_to_delete) + + assert not path2.exists(), "Video 2 should be deleted from filesystem" + assert path1.exists(), "Video 1 should still exist" + assert path3.exists(), "Video 3 should still exist" + + # Build move mappings for all 3 videos (as execute_reorganization would) + dest1 = channel_dir / 'reorganized_video_1.mp4' + dest2 = channel_dir / 'reorganized_deleted.mp4' + dest3 = channel_dir / 'reorganized_video_3.mp4' + + move_mappings = [ + (path1, dest1), + (path2, dest2), # This file doesn't exist - should be skipped + (path3, dest3), + ] + + config = get_videos_downloader_config() + task = FileTask( + task_type=FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + collection_id=collection_id, + job_id='test-deleted-files', + pending_file_format=config.file_name_format, + ) + + # This should NOT raise an exception + await file_worker.handle_reorganize(task) + await asyncio.sleep(0.1) + + # Verify results + with get_db_session() as session: + # Video 1: Should be moved and DB updated + fg1 = session.query(FileGroup).get(fg1_id) + assert fg1.primary_path == dest1, f"Video 1 should be moved to dest. Got {fg1.primary_path}" + assert dest1.exists(), "Video 1 file should exist at destination" + assert not path1.exists(), "Video 1 file should not exist at source" + + # Video 2: Was deleted from both filesystem and DB, move_mapping was skipped + # (The reorganization logged a warning and continued) + assert not path2.exists(), "Video 2 file should not exist at source (was deleted)" + assert not dest2.exists(), "Video 2 file should not exist at destination (was deleted)" + + # Video 3: Should be moved and DB updated + fg3 = session.query(FileGroup).get(fg3_id) + assert fg3.primary_path == dest3, f"Video 3 should be moved to dest. Got {fg3.primary_path}" + assert dest3.exists(), "Video 3 file should exist at destination" + assert not path3.exists(), "Video 3 file should not exist at source" + + # Collection file_format should be updated (reorganization completed successfully) + collection = session.query(Collection).get(collection_id) + assert collection.file_format == config.file_name_format, \ + "Collection file_format should be updated after successful reorganization" + + +@pytest.mark.parametrize("file_format,has_upload_date,has_source_id,has_uploader,should_produce_path", [ + # Format requires upload_year - needs upload_date + ("%(upload_year)s/%(title)s.%(ext)s", False, False, False, False), + ("%(upload_year)s/%(title)s.%(ext)s", True, False, False, True), + # Format requires source_id - needs source_id + ("%(id)s_%(title)s.%(ext)s", False, False, False, False), + ("%(id)s_%(title)s.%(ext)s", False, True, False, True), + # Format requires uploader - needs uploader + ("%(uploader)s_%(title)s.%(ext)s", False, False, False, False), + ("%(uploader)s_%(title)s.%(ext)s", False, False, True, True), + # Format only needs title - always works + ("%(title)s.%(ext)s", False, False, False, True), + ("%(title)s.%(ext)s", True, True, True, True), + # Complex format - needs all metadata (upload_date, source_id, uploader) + ("%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s", False, False, False, False), + ("%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s", True, True, True, True), + # Missing some but has others - still fails if format requires the missing one + ("%(upload_year)s/%(uploader)s_%(title)s.%(ext)s", False, True, True, False), # missing upload_date + ("%(upload_year)s/%(uploader)s_%(title)s.%(ext)s", True, False, False, False), # missing uploader + ("%(upload_year)s/%(uploader)s_%(title)s.%(ext)s", True, False, True, True), # has both required +]) +def test_reorganize_skips_videos_missing_required_metadata( + test_session, test_directory, video_factory, + file_format, has_upload_date, has_source_id, has_uploader, should_produce_path +): + """Videos should only be skipped when missing metadata that the format requires. + + This test verifies that _compute_new_path_for_video correctly determines + whether a video has sufficient metadata based on the configured format string. + """ + from datetime import datetime, timezone + from wrolpi.collections.reorganize import _compute_new_path_for_video + + # Create channel collection + channel_dir = test_directory / 'videos' / 'MetadataTestChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + collection = Collection( + name='MetadataTestChannel', + kind='channel', + directory=channel_dir, + ) + test_session.add(collection) + test_session.flush() + channel = Channel(name='MetadataTestChannel', collection_id=collection.id, directory=channel_dir) + test_session.add(channel) + test_session.commit() + channel_id = channel.id + + # Build info_json dict if we need uploader or source_id from it + info_json = {} + if has_uploader: + info_json['uploader'] = 'TestUploader' + if has_source_id: + info_json['id'] = 'abc123' + + # Create video with controlled metadata + # with_info_json creates the actual .info.json file on disk + video = video_factory( + channel_id=channel_id, + title='Test Video', + with_video_file=True, + with_info_json=info_json if info_json else None, + ) + + video_obj = test_session.query(Video).get(video.id) + + # Set source_id directly on the Video model (separate from info_json) + video_obj.source_id = 'abc123' if has_source_id else None + video_obj.file_group.published_datetime = datetime(2024, 6, 15, tzinfo=timezone.utc) if has_upload_date else None + + test_session.commit() + test_session.refresh(video_obj) + + # Call _compute_new_path_for_video with explicit file_format + new_path = _compute_new_path_for_video(video_obj, channel_dir, file_format=file_format) + + if should_produce_path: + assert new_path is not None, ( + f"Format '{file_format}' with upload_date={has_upload_date}, " + f"source_id={has_source_id}, uploader={has_uploader} should produce a path" + ) + # Verify the path doesn't contain empty segments + path_str = str(new_path.relative_to(channel_dir)) + assert not path_str.startswith('/'), f"Path should not start with /: {path_str}" + else: + assert new_path is None, ( + f"Format '{file_format}' with upload_date={has_upload_date}, " + f"source_id={has_source_id}, uploader={has_uploader} should return None, " + f"but got: {new_path}" + ) + + +@pytest.mark.parametrize("file_format,has_url,should_produce_path", [ + # Format requires domain - needs URL + ("%(domain)s/%(title)s.%(ext)s", False, False), + ("%(domain)s/%(title)s.%(ext)s", True, True), + # Format only needs title/download_date - always works + ("%(title)s.%(ext)s", False, True), + ("%(title)s.%(ext)s", True, True), + ("%(download_datetime)s_%(title)s.%(ext)s", False, True), + ("%(download_datetime)s_%(title)s.%(ext)s", True, True), + # Complex format with domain + ("%(download_year)s/%(domain)s_%(title)s.%(ext)s", False, False), + ("%(download_year)s/%(domain)s_%(title)s.%(ext)s", True, True), + # Format without domain - always works + ("%(download_year)s/%(download_month)s/%(title)s.%(ext)s", False, True), +]) +def test_reorganize_skips_archives_missing_required_metadata( + test_session, test_directory, archive_factory, + file_format, has_url, should_produce_path +): + """Archives should only be skipped when missing metadata that the format requires. + + Archives have robust fallbacks for most variables (dates fallback to now(), + title to 'untitled'). Only domain can be empty when URL is missing. + """ + from wrolpi.collections.reorganize import _compute_new_path_for_archive + + # Create domain collection + domain_dir = test_directory / 'archive' / 'test.com' + domain_dir.mkdir(parents=True, exist_ok=True) + + collection = Collection( + name='test.com', + kind='domain', + directory=domain_dir, + ) + test_session.add(collection) + test_session.commit() + + # Create archive with controlled metadata + archive = archive_factory( + domain='test.com', + url='https://test.com/article' if has_url else None, + title='Test Article', + ) + + archive_obj = test_session.query(Archive).get(archive.id) + + # Clear URL if testing without it + if not has_url: + archive_obj.file_group.url = None + test_session.commit() + test_session.refresh(archive_obj) + + # Call _compute_new_path_for_archive with explicit file_format + new_path = _compute_new_path_for_archive(archive_obj, domain_dir, file_format=file_format) + + if should_produce_path: + assert new_path is not None, ( + f"Format '{file_format}' with url={has_url} should produce a path" + ) + else: + assert new_path is None, ( + f"Format '{file_format}' with url={has_url} should return None, " + f"but got: {new_path}" + ) + + +# ============================================================================ +# Partial Reorganization Tests (Individual Collection Reorganization) +# ============================================================================ + + +@pytest.mark.asyncio +async def test_domain_reorganization_moves_files_from_root_to_year_subdirectory( + async_client, test_directory, archive_factory +): + """Archives in collection root should be moved to year subdirectories. + + Scenario: Batch reorganization completed but missed some files. + - collection.file_format matches config (needs_reorganization=False) + - Some archives are in root: domain.com/2000-01-01_article.html + - Format expects: domain.com/2000/2000-01-01_article.html + - Reorganization should move files from root to year subdirectory + """ + from wrolpi.files.worker import file_worker + + # Create domain collection with year-based format + domain_dir = test_directory / 'archive' / 'test.com' + domain_dir.mkdir(parents=True, exist_ok=True) + + year_format = '%(download_year)s/%(download_datetime)s_%(title)s.%(ext)s' + + # Temporarily change config to use year format + config = get_archive_downloader_config() + original_format = config._config['file_name_format'] + config._config['file_name_format'] = year_format + + try: + with get_db_session(commit=True) as session: + collection = Collection( + name='test.com', + kind='domain', + directory=domain_dir, + file_format=year_format, # Matches expected format (appears organized) + ) + session.add(collection) + session.commit() + collection_id = collection.id + + # Create archive using factory - files will be in root (no year subdir) + # Factory creates files with format: %(download_datetime)s_%(title)s.%(ext)s (NOT year) + archive = archive_factory(domain='test.com', url='https://test.com/article', title='Test Article') + + with get_db_session(commit=True) as session: + archive_obj = session.query(Archive).get(archive.id) + archive_obj.collection_id = collection_id + session.commit() + + # Verify: collection appears organized but files are in wrong place + collection = session.query(Collection).get(collection_id) + assert collection.file_format == year_format, "Collection format should match config" + assert collection.needs_reorganization is False, \ + "Collection should appear organized (format matches config)" + + old_path = archive_obj.file_group.primary_path + assert old_path.parent == domain_dir, \ + f"Archive should be in root, not year subdir. Got: {old_path.parent}" + + # Get preview with exact_count - should detect files need moving + with get_db_session() as session: + preview = get_reorganization_preview(collection_id, session, exact_count=True) + assert preview.files_needing_move > 0, \ + f"Should detect files need moving. Got: {preview.files_needing_move}" + + # Execute reorganization + with get_db_session(commit=True) as session: + job_id = execute_reorganization(collection_id, session) + assert job_id, "Should return job_id for files to move" + + # Process the reorganization + file_worker.transfer_queue() + await file_worker.process_queue() + + # Verify: files moved to year subdirectory + with get_db_session() as session: + archive_obj = session.query(Archive).get(archive.id) + new_path = archive_obj.file_group.primary_path + assert '/2000/' in str(new_path), f"File should be in year subdirectory. Got: {new_path}" + assert new_path.exists(), "New file should exist" + finally: + config._config['file_name_format'] = original_format + + +@pytest.mark.asyncio +async def test_channel_reorganization_moves_files_from_root_to_year_subdirectory( + async_client, test_directory, video_factory +): + """Videos in collection root should be moved to year subdirectories. + + Scenario: Files are in the wrong place and need to be moved. + - Some videos are in root: channel/title.mp4 + - Format expects: channel/2024/title.mp4 + - Reorganization should move files from root to year subdirectory + """ + from datetime import datetime, timezone + from wrolpi.files.worker import file_worker + + # Create channel collection with year-based format + channel_dir = test_directory / 'videos' / 'TestChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + year_format = '%(upload_year)s/%(title)s.%(ext)s' + + # Temporarily change config to use year format (videos config is nested) + # Manager dict requires reassigning the entire nested dict for changes to propagate + config = get_videos_downloader_config() + original_format = config._config['yt_dlp_options']['file_name_format'] + yt_dlp_options = dict(config._config['yt_dlp_options']) + yt_dlp_options['file_name_format'] = year_format + config._config['yt_dlp_options'] = yt_dlp_options + + try: + with get_db_session(commit=True) as session: + collection = Collection( + name='TestChannel', + kind='channel', + directory=channel_dir, + file_format=year_format, + ) + session.add(collection) + session.flush() + channel = Channel(name='TestChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + collection_id = collection.id + channel_id = channel.id + + # Create video using factory - files will be in root (no year subdir) + video = video_factory( + channel_id=channel_id, + title='test_video', + with_video_file=True, + with_info_json={'uploader': 'Tester', 'id': 'abc123'}, + upload_date=datetime(2024, 6, 15, tzinfo=timezone.utc), + ) + + with get_db_session(commit=True) as session: + video_obj = session.query(Video).get(video.id) + + old_path = video_obj.file_group.primary_path + assert old_path.parent == channel_dir, \ + f"Video should be in root, not year subdir. Got: {old_path.parent}" + + # Get preview with exact_count - should detect files need moving + with get_db_session() as session: + preview = get_reorganization_preview(collection_id, session, exact_count=True) + assert preview.files_needing_move > 0, \ + f"Should detect files need moving. Got: {preview.files_needing_move}" + + # Execute reorganization + with get_db_session(commit=True) as session: + job_id = execute_reorganization(collection_id, session) + assert job_id, "Should return job_id for files to move" + + # Process the reorganization + file_worker.transfer_queue() + await file_worker.process_queue() + + # Verify: files moved to year subdirectory + with get_db_session() as session: + video_obj = session.query(Video).get(video.id) + new_path = video_obj.file_group.primary_path + assert '/2024/' in str(new_path), f"File should be in year subdirectory. Got: {new_path}" + assert new_path.exists(), "New file should exist" + finally: + yt_dlp_options = dict(config._config['yt_dlp_options']) + yt_dlp_options['file_name_format'] = original_format + config._config['yt_dlp_options'] = yt_dlp_options + + +def test_preview_exact_count_computes_actual_moves(test_session, test_directory, archive_factory): + """With exact_count=True, preview should compute actual files needing move, not total files. + + This test verifies that: + - exact_count=False returns total_files as files_needing_move (estimate) + - exact_count=True computes the actual count by building move mappings + """ + # Create domain collection with a year-based format (different from default) + domain_dir = test_directory / 'archive' / 'exactcount.com' + domain_dir.mkdir(parents=True, exist_ok=True) + + year_format = '%(download_year)s/%(download_datetime)s_%(title)s.%(ext)s' + + # Temporarily change config to use year format + config = get_archive_downloader_config() + original_format = config._config['file_name_format'] + config._config['file_name_format'] = year_format + + try: + collection = Collection( + name='exactcount.com', + kind='domain', + directory=domain_dir, + file_format=year_format, # Matches config + ) + test_session.add(collection) + test_session.commit() + collection_id = collection.id + + # Create archive using factory - files will be in root (need to move to year subdir) + archive = archive_factory(domain='exactcount.com', url='https://exactcount.com/article', title='Test') + + with get_db_session(commit=True) as session: + archive_obj = session.query(Archive).get(archive.id) + archive_obj.collection_id = collection_id + session.commit() + + # Without exact_count, preview assumes all files need moving + with get_db_session() as session: + preview_estimate = get_reorganization_preview(collection_id, session, exact_count=False) + # This is the current behavior - assumes all need moving + assert preview_estimate.files_needing_move == preview_estimate.total_files + assert preview_estimate.files_needing_move == 1 + + # With exact_count, preview also detects that files need moving (they're in wrong location) + with get_db_session() as session: + preview_exact = get_reorganization_preview(collection_id, session, exact_count=True) + # Files are in root, but should be in year subdir, so 1 file needs moving + assert preview_exact.files_needing_move == 1, \ + f"File in wrong location should need moving. Got: {preview_exact.files_needing_move}" + + # Now execute the reorganization to move files to correct location + from wrolpi.files.worker import file_worker + with get_db_session(commit=True) as session: + job_id = execute_reorganization(collection_id, session) + assert job_id, "Should return job_id" + + file_worker.transfer_queue() + import asyncio + asyncio.get_event_loop().run_until_complete(file_worker.process_queue()) + + # After reorganization, with exact_count should show 0 files needing move + with get_db_session() as session: + preview_after = get_reorganization_preview(collection_id, session, exact_count=True) + assert preview_after.files_needing_move == 0, \ + f"After reorganization, no files should need moving. Got: {preview_after.files_needing_move}" + finally: + config._config['file_name_format'] = original_format + + +@pytest.mark.asyncio +async def test_reorganize_fails_on_filename_conflicts(async_client, test_directory, video_factory): + """Reorganization should fail early when two videos would have the same destination filename.""" + from datetime import datetime, timezone + + # Create channel collection + channel_dir = test_directory / 'videos' / 'ConflictChannel' + channel_dir.mkdir(parents=True, exist_ok=True) + + with get_db_session(commit=True) as session: + collection = Collection( + name='ConflictChannel', + kind='channel', + directory=channel_dir, + file_format='%(old_format)s.%(ext)s' # Needs reorganization + ) + session.add(collection) + session.flush() + channel = Channel(name='ConflictChannel', collection_id=collection.id, directory=channel_dir) + session.add(channel) + session.commit() + channel_id = channel.id + collection_id = collection.id + + # Create two videos with DIFFERENT initial filenames but IDENTICAL destination metadata + # Both videos will have the same uploader, upload_date, source_id, and title + # This should produce a filename conflict + same_date = datetime(2024, 1, 15, tzinfo=timezone.utc) + same_source_id = 'duplicate_id' + + video1 = video_factory( + channel_id=channel_id, + title='video_one', # Different initial filename + upload_date=same_date, + with_video_file=True, + with_info_json={'uploader': 'TestUploader', 'id': same_source_id} + ) + + video2 = video_factory( + channel_id=channel_id, + title='video_two', # Different initial filename + upload_date=same_date, + with_video_file=True, + with_info_json={'uploader': 'TestUploader', 'id': same_source_id} + ) + + # Update both videos to have the same title and source_id + # This makes them produce the same destination filename + with get_db_session(commit=True) as session: + v1 = session.query(Video).get(video1.id) + v2 = session.query(Video).get(video2.id) + v1.file_group.title = 'Duplicate Title' + v2.file_group.title = 'Duplicate Title' + v1.source_id = same_source_id + v2.source_id = same_source_id + + # Attempting to reorganize should fail with conflict error + with pytest.raises(ReorganizationConflict) as exc_info: + with get_db_session() as session: + execute_reorganization(collection_id, session) + + # Error message should mention the conflict + assert 'conflict' in str(exc_info.value).lower() + assert 'Duplicate Title' in str(exc_info.value) + diff --git a/wrolpi/common.py b/wrolpi/common.py index e96531c6..fe18f30a 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1404,27 +1404,45 @@ def escape_file_name(name: str) -> str: return name.strip() -# Maximum length is probably 255, but we need more length for large suffixes like `.readability.json`, and temporary -# downloading suffixes from yt-dlp. -MAXIMUM_FILE_LENGTH = 140 +# Maximum bytes for a filename. Linux ext4/xfs limit is 255 bytes. +# We use 200 to leave room for FileGroup suffix variations (e.g., .readability.html is 17 bytes vs .html at 5 bytes). +MAXIMUM_FILE_BYTES = 200 def trim_file_name(path: str | pathlib.Path) -> str | pathlib.Path: """Shorten the file name only if it is longer than the file system supports. Trim from the end of the name until - the name is short enough (preserving any suffix).""" + the name is short enough (preserving any suffix). + + Uses BYTE length (not character length) because UTF-8 multi-byte characters (e.g., Devanagari, Chinese) + can exceed the 255 byte filesystem limit even with fewer than 140 characters. + """ name_type = pathlib.Path if isinstance(path, pathlib.Path) else str parent = path.parent if isinstance(path, pathlib.Path) else '/'.join(i for i in path.split('/')[:-1]) - path = path.name if isinstance(path, pathlib.Path) else path + filename = path.name if isinstance(path, pathlib.Path) else path.split('/')[-1] if '/' in path else path - if len(path) < MAXIMUM_FILE_LENGTH: - return name_type(path) + filename_bytes = len(filename.encode('utf-8')) + if filename_bytes <= MAXIMUM_FILE_BYTES: + # No trimming needed, but preserve parent directory if present + if parent: + if name_type == pathlib.Path: + return parent / filename + return f'{parent}/{filename}' + return name_type(filename) - # Don't trim the filename to exactly 256 characters. + # Trim based on byte length, not character length. # This is because a FileGroup will have varying filename lengths. from wrolpi.files.lib import split_path_stem_and_suffix - stem, suffix = split_path_stem_and_suffix(path) - excess = MAXIMUM_FILE_LENGTH - len(suffix) - new_name = stem[:excess].strip() + suffix + stem, suffix = split_path_stem_and_suffix(filename) + suffix_bytes = len(suffix.encode('utf-8')) + target_stem_bytes = MAXIMUM_FILE_BYTES - suffix_bytes + + # Trim characters from end of stem until byte length fits + trimmed_stem = stem + while len(trimmed_stem.encode('utf-8')) > target_stem_bytes and trimmed_stem: + trimmed_stem = trimmed_stem[:-1] + trimmed_stem = trimmed_stem.strip() + + new_name = trimmed_stem + suffix if parent: if name_type == pathlib.Path: return parent / new_name @@ -1573,7 +1591,22 @@ def register_modeler(modeler: callable): return modeler -async def apply_modelers(): +async def apply_modelers(progress_callback: callable = None): + """Run all registered modelers. + + Args: + progress_callback: Optional callback(processed, total) called after each modeler completes. + """ + from wrolpi.db import get_db_session + from wrolpi.files.models import FileGroup + + # Get initial count of unindexed files for progress tracking + initial_unindexed = 0 + if progress_callback: + with get_db_session() as session: + initial_unindexed = session.query(FileGroup).filter(FileGroup.indexed != True).count() + progress_callback(0, initial_unindexed) + for modeler in modelers: logger_.info(f'Applying modeler {modeler.__name__}') try: @@ -1582,6 +1615,14 @@ async def apply_modelers(): logger_.error(f'Modeler {modeler.__name__} raised error', exc_info=e) if PYTEST: raise + + # Update progress after each modeler + if progress_callback and initial_unindexed > 0: + with get_db_session() as session: + remaining = session.query(FileGroup).filter(FileGroup.indexed != True).count() + processed = initial_unindexed - remaining + progress_callback(processed, initial_unindexed) + # Sleep to catch cancel. await asyncio.sleep(0) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 494faa85..b9c6251a 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -12,7 +12,7 @@ import subprocess import urllib.parse from itertools import zip_longest from pathlib import Path -from typing import List, Tuple, Union, Dict, Generator, Iterable, Set +from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set import cachetools.func import psycopg2 @@ -724,11 +724,23 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim curs.execute(stmt) -async def apply_indexers(): - """Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function.""" +async def apply_indexers(progress_callback: Callable[[int, int], None] = None): + """Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function. + + Args: + progress_callback: Optional callback(processed, total) called after each batch. + """ from wrolpi.files.models import FileGroup refresh_logger.info('Applying indexers') + # Get initial count for progress tracking + total_to_index = 0 + total_indexed = 0 + if progress_callback: + with get_db_session() as session: + total_to_index = session.query(FileGroup).filter(FileGroup.indexed != True).count() + progress_callback(0, total_to_index) + while True: # Continually query for Files that have not been indexed. with get_db_session(commit=False) as session: @@ -809,6 +821,11 @@ async def apply_indexers(): if file_group: refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}') + # Update progress + total_indexed += processed + if progress_callback and total_to_index > 0: + progress_callback(total_indexed, total_to_index) + if processed < 20: # Processed less than the limit, don't do the next query. break @@ -1365,6 +1382,51 @@ def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path, curs.execute(sql) +def _json_serial(obj): + """JSON serializer for objects not serializable by default json code.""" + import datetime + if isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + raise TypeError(f"Type {type(obj)} not serializable") + + +def _bulk_update_file_groups_reorganize(updates: List[dict]): + """Batch UPDATE FileGroups for reorganization using raw SQL. + + Unlike _bulk_update_file_groups_db, this also updates the files and data JSON fields + since reorganization changes filenames, not just directories. + + Args: + updates: List of dicts with keys: id, directory, primary_path, files, data + """ + if not updates: + return + + with get_db_curs(commit=True) as curs: + for update in updates: + # Don't change indexed - reorganization only moves files, content unchanged + curs.execute(''' + UPDATE file_group + SET directory = %s, + primary_path = %s, + files = %s, + data = %s + WHERE id = %s + ''', ( + update['directory'], + update['primary_path'], + json.dumps(update['files'], default=_json_serial), + json.dumps(update['data'], default=_json_serial) if update.get('data') else None, + update['id'], + )) + + # Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates. + # This is necessary because get_db_curs uses the same connection as get_db_session + # during tests, but SQLAlchemy's identity map doesn't know about raw SQL updates. + with get_db_session() as session: + session.expire_all() + + def delete_directory(directory: pathlib.Path, recursive: bool = False): """Remove a directory, remove it's Directory record. diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 4ba57f65..86c4e7a9 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -1360,3 +1360,51 @@ async def test_rename_non_primary_file_renames_filegroup(async_client, test_sess assert 'renamed.mp4' in file_names assert 'renamed.srt' in file_names assert len(file_group.files) == 2 + + + +def test_bulk_update_file_groups_reorganize_handles_datetime(test_session, test_directory): + """_bulk_update_file_groups_reorganize should handle datetime objects in files JSON. + + The files JSON field can contain modification_datetime as a datetime object. + The function must serialize these correctly to avoid JSON serialization errors. + """ + from datetime import datetime, timezone + from wrolpi.files.lib import _bulk_update_file_groups_reorganize + + # Create a test file and FileGroup + test_file = test_directory / 'test_video.mp4' + test_file.touch() + + fg = FileGroup() + fg.directory = test_directory + fg.primary_path = test_file + fg.files = [{'path': 'test_video.mp4', 'mimetype': 'video/mp4'}] + test_session.add(fg) + test_session.commit() + fg_id = fg.id + + # Prepare update with datetime object in files (simulating what handle_reorganize does) + new_files = [{ + 'path': 'test_video.mp4', + 'mimetype': 'video/mp4', + 'modification_datetime': datetime.now(timezone.utc), # datetime object, not string + }] + + updates = [{ + 'id': fg_id, + 'directory': str(test_directory), + 'primary_path': str(test_file), + 'files': new_files, + }] + + # This should NOT raise "Object of type datetime is not JSON serializable" + _bulk_update_file_groups_reorganize(updates) + + # Verify the update was applied + test_session.expire_all() + fg = test_session.query(FileGroup).filter_by(id=fg_id).one() + assert len(fg.files) == 1 + assert fg.files[0]['path'] == 'test_video.mp4' + # modification_datetime should be serialized as ISO string + assert 'modification_datetime' in fg.files[0] diff --git a/wrolpi/files/test/test_worker.py b/wrolpi/files/test/test_worker.py index 6b36e750..b0188d05 100644 --- a/wrolpi/files/test/test_worker.py +++ b/wrolpi/files/test/test_worker.py @@ -302,10 +302,11 @@ async def test_compare_file_groups_ignores_outside_root(test_session, test_direc @pytest.mark.asyncio async def test_compare_file_groups_cancelable(test_session, test_directory, make_files_structure): - """compare_file_groups can be canceled during long operations.""" + """compare_file_groups can be canceled during long operations and reports progress via callback.""" # Create many files to ensure scan takes time make_files_structure([f'docs/file{i}.txt' for i in range(100)]) + # Test cancellation task = asyncio.create_task(compare_file_groups(test_directory)) await asyncio.sleep(0) # Let task start task.cancel() @@ -313,6 +314,13 @@ async def test_compare_file_groups_cancelable(test_session, test_directory, make with pytest.raises(asyncio.CancelledError): await task + # Test progress callback with small batch size + progress_counts = [] + await compare_file_groups(test_directory, batch_size=10, progress_callback=progress_counts.append) + # Should have been called multiple times with increasing counts + assert len(progress_counts) >= 1 + assert progress_counts == sorted(progress_counts) # Counts should be increasing + @pytest.mark.asyncio async def test_count_files_empty(test_directory): diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index 3eeef97e..c489dacf 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -30,6 +30,7 @@ from wrolpi.files.lib import ( split_path_stem_and_suffix, _upsert_files, get_unique_files_by_stem, glob_shared_stem, group_files_by_stem, get_primary_file, delete_directory, apply_indexers, _move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE, + _bulk_update_file_groups_reorganize, ) logger = logger.getChild(__name__) @@ -274,7 +275,11 @@ def _insert_fs_batch(session, batch: list): ) -async def compare_file_groups(root: Path = None, batch_size: int = 10000) -> FileComparisonResult: +async def compare_file_groups( + root: Path = None, + batch_size: int = 10000, + progress_callback: Callable[[int], None] = None, +) -> FileComparisonResult: """ Compare FileGroups in DB to filesystem. Memory efficient for Raspberry Pi. @@ -317,6 +322,8 @@ async def compare_file_groups(root: Path = None, batch_size: int = 10000) -> Fil _insert_fs_batch(session, batch) file_count += len(batch) batch = [] + if progress_callback: + progress_callback(file_count) await asyncio.sleep(0) # Cancellation yield point finally: await path_gen.aclose() # Ensure subprocess is killed on cancellation @@ -691,6 +698,8 @@ class FileTaskType(str, Enum): move = auto() # Move files and directories to a new location, file names are not changed. rename = auto() # Rename files, requires deeper updating of FileGroup.files and FileGroup.data. tag = auto() # Add TagFile records for files. + reorganize = auto() # Reorganize files with arbitrary source->dest mappings for collection reorganization. + batch_reorganize = auto() # Batch reorganize multiple collections sequentially. @dataclass @@ -708,6 +717,15 @@ class FileTask: next_task_type: FileTaskType = None job_id: str = None # Unique ID for tracking completion expand_stems: bool = True # Whether to expand files to their FileGroup stem-mates + # For reorganize tasks: list of (source_path, dest_path) tuples + move_mappings: list[tuple[pathlib.Path, pathlib.Path]] = None + collection_id: int = None # Collection being reorganized + collection_kind: str = None # 'channel' or 'domain' for reorganize tasks + # For batch_reorganize tasks + collection_ids: list[int] = None # List of collection IDs to process + batch_kind: str = None # 'channel' or 'domain' + # For deferred file_format update (only updated after successful completion) + pending_file_format: str = None class FileWorker: @@ -844,6 +862,75 @@ class FileWorker: self.public_queue.put_nowait(task) return job_id + def queue_reorganize( + self, + move_mappings: list[tuple[pathlib.Path, pathlib.Path]], + collection_id: int = None, + collection_kind: str = None, + pending_file_format: str = None, + ) -> str: + """Queue a reorganize task for collection file reorganization. + + Unlike queue_move which moves all files to a single destination, + queue_reorganize handles arbitrary source->destination mappings + where each file can move to a different location. + + Args: + move_mappings: List of (source_path, dest_path) tuples + collection_id: Optional collection ID being reorganized (for status tracking) + collection_kind: 'channel' or 'domain' (for status tracking / navbar navigation) + pending_file_format: File format to set on collection after successful completion + (deferred update enables retry on failure) + + Returns: + Job ID for tracking completion + """ + import uuid + job_id = f'reorganize-{uuid.uuid4().hex[:8]}' + # Paths list is empty for reorganize tasks - we use move_mappings instead + task = FileTask( + FileTaskType.reorganize, + paths=[], + move_mappings=move_mappings, + collection_id=collection_id, + collection_kind=collection_kind, + job_id=job_id, + pending_file_format=pending_file_format, + ) + self._set_job_status(job_id, 'pending') + self.public_queue.put_nowait(task) + return job_id + + def queue_batch_reorganize( + self, + collection_ids: list[int], + kind: str, + ) -> str: + """Queue a batch reorganize task for multiple collections. + + Processes collections sequentially, stopping on first failure. + Tracks overall progress and per-collection progress. + + Args: + collection_ids: List of collection IDs to reorganize + kind: 'channel' or 'domain' + + Returns: + Batch job ID for tracking completion + """ + import uuid + job_id = f'batch-reorganize-{uuid.uuid4().hex[:8]}' + task = FileTask( + FileTaskType.batch_reorganize, + paths=[], + collection_ids=collection_ids, + batch_kind=kind, + job_id=job_id, + ) + self._set_job_status(job_id, 'pending') + self.public_queue.put_nowait(task) + return job_id + async def refresh_sync(self, paths: list[pathlib.Path]): """Synchronously refresh specific files. Use sparingly - prefer queue_refresh.""" if not paths: @@ -864,17 +951,32 @@ class FileWorker: try: task: FileTask = self.private_queue.get_nowait() except asyncio.QueueEmpty: - # No file tasks to perform. Sleep to catch cancel. + # No tasks - clear flag if set and both queues are empty + if flags.file_worker_busy.is_set() and self.public_queue.qsize() == 0: + flags.file_worker_busy.clear() await asyncio.sleep(0) return - match task.task_type: - case FileTaskType.count: - await self.handle_count(task) - case FileTaskType.refresh: - await self.handle_refresh(task) - case FileTaskType.move: - await self.handle_move(task) + # Set flag before processing any task (if not already set) + if not flags.file_worker_busy.is_set(): + flags.file_worker_busy.set() + + try: + match task.task_type: + case FileTaskType.count: + await self.handle_count(task) + case FileTaskType.refresh: + await self.handle_refresh(task) + case FileTaskType.move: + await self.handle_move(task) + case FileTaskType.reorganize: + await self.handle_reorganize(task) + case FileTaskType.batch_reorganize: + await self.handle_batch_reorganize(task) + finally: + # Clear flag if both queues are now empty + if self.private_queue.qsize() == 0 and self.public_queue.qsize() == 0: + flags.file_worker_busy.clear() async def handle_count(self, task: FileTask): """Count files in the task's paths and chain to next task if specified.""" @@ -1006,8 +1108,12 @@ class FileWorker: ) logger.info(f'Comparing {task.count} files in {len(dir_paths)} directories') + def on_compare_progress(count: int): + percent = int((count / task.count) * 100) if task.count > 0 else 0 + self.update_status(operation_processed=count, operation_percent=percent) + root = dir_paths[0] if len(dir_paths) == 1 else None - dir_result = await compare_file_groups(root=root) + dir_result = await compare_file_groups(root=root, progress_callback=on_compare_progress) if is_global_refresh: Events.send_global_refresh_discovery_completed() @@ -1661,15 +1767,23 @@ class FileWorker: """ from wrolpi.tags import save_tags_config - self.update_status(status='modeling') + def on_modeling_progress(processed: int, total: int): + percent = int((processed / total) * 100) if total > 0 else 0 + self.update_status(operation_total=total, operation_processed=processed, operation_percent=percent) + + self.update_status(status='modeling', operation_total=0, operation_processed=0, operation_percent=0) with flags.file_worker_modeling: - await apply_modelers() + await apply_modelers(progress_callback=on_modeling_progress) if is_global_refresh: Events.send_global_refresh_modeling_completed() - self.update_status(status='indexing') + def on_indexing_progress(processed: int, total: int): + percent = int((processed / total) * 100) if total > 0 else 0 + self.update_status(operation_total=total, operation_processed=processed, operation_percent=percent) + + self.update_status(status='indexing', operation_total=0, operation_processed=0, operation_percent=0) with flags.file_worker_indexing: - await apply_indexers() + await apply_indexers(progress_callback=on_indexing_progress) if is_global_refresh: Events.send_global_refresh_indexing_completed() @@ -1763,27 +1877,26 @@ class FileWorker: plan = {} try: - with flags.file_worker_busy: - destination.mkdir(parents=True, exist_ok=True) + destination.mkdir(parents=True, exist_ok=True) - # Build the move plan using bulk SQL operations - plan, old_directories = await build_move_plan_bulk(sources, destination) + # Build the move plan using bulk SQL operations + plan, old_directories = await build_move_plan_bulk(sources, destination) - # Update status with total - self.update_status( - status='moving', - operation_total=len(plan), - ) + # Update status with total + self.update_status( + status='moving', + operation_total=len(plan), + ) - # Execute the plan in chunks - with flags.file_worker_discovery: - with get_db_session(commit=True) as session: - await self._execute_move_chunks( - plan, session, created_directories, revert_plan - ) + # Execute the plan in chunks + with flags.file_worker_discovery: + with get_db_session(commit=True) as session: + await self._execute_move_chunks( + plan, session, created_directories, revert_plan + ) - self._cleanup_old_directories(sources, old_directories) - await self._apply_post_processing() + self._cleanup_old_directories(sources, old_directories) + await self._apply_post_processing() logger.info(f'Move completed: {len(plan)} items moved to {destination}') Events.send_file_move_completed(f'Moved {len(sources)} items to {destination}') @@ -1800,6 +1913,499 @@ class FileWorker: finally: self.reset_status() + async def handle_reorganize(self, task: FileTask): + """Handle reorganize task for collection file reorganization. + + Unlike handle_move which moves all files to a single destination, + handle_reorganize handles arbitrary source->destination mappings + where each file (or FileGroup) can move to a different location. + + This is used when a user changes their file_name_format configuration + and wants to reorganize existing files to match the new format. + + Optimized for bulk operations: + - Single query to pre-fetch all FileGroups + - Uses FileGroup.files instead of filesystem globs + - Dict lookup for O(n) path matching + - Batched database updates + """ + from wrolpi.events import Events + from wrolpi.files.models import FileGroup + + move_mappings = task.move_mappings + + if not move_mappings: + logger.info('Reorganize task has no move mappings, nothing to do') + self._complete_job(task.job_id) + self.reset_status() + return + + total_moves = len(move_mappings) + logger.info(f'Starting reorganize task with {total_moves} file moves') + + # Transition job from 'pending' to 'running' so progress can be tracked + self._set_job_status(task.job_id, 'running') + + self.update_status( + status='reorganizing', + task_type='reorganize', + paths=[], + destination=None, + operation_total=total_moves, + operation_processed=0, + operation_percent=0, + error=None, + collection_id=task.collection_id, + collection_kind=task.collection_kind, + ) + + # Track successful moves for rollback + completed_moves: list[tuple[pathlib.Path, pathlib.Path]] = [] + # Track created directories for cleanup + created_directories: set[pathlib.Path] = set() + # Track source directories to clean up after moves + source_directories: set[pathlib.Path] = set() + + try: + with flags.file_worker_discovery: + # Phase 1: Bulk pre-fetch all FileGroups + source_paths = [str(s) for s, d in move_mappings] + with get_db_session() as session: + file_groups = session.query(FileGroup).filter( + FileGroup.primary_path.in_(source_paths) + ).all() + # Build lookup dict and detach from session + # Note: fg.primary_path is a pathlib.Path (from MediaPathType), but + # source_paths are strings, so we use str() for the dictionary keys + fg_by_path = {} + for fg in file_groups: + fg_by_path[str(fg.primary_path)] = { + 'id': fg.id, + 'directory': pathlib.Path(fg.directory), + 'files': list(fg.files), # Copy the list + 'data': dict(fg.data) if fg.data else None, + } + + processed = 0 + pending_updates = [] + + for source_path, dest_path in move_mappings: + source_path = pathlib.Path(source_path) + dest_path = pathlib.Path(dest_path) + + source_exists = source_path.exists() + dest_exists = dest_path.exists() + + # Look up the pre-fetched FileGroup + fg_info = fg_by_path.get(str(source_path)) + if not fg_info: + logger.debug(f'FileGroup not found by primary_path: {source_path}') + processed += 1 + continue + + if source_exists and dest_exists: + # Conflict: both exist - skip with warning + logger.warning(f'Reorganize: both source and destination exist: {source_path} -> {dest_path}') + processed += 1 + continue + + if not source_exists and not dest_exists: + # Neither exists - skip with warning + logger.warning(f'Reorganize: neither source nor destination exist: {source_path}') + processed += 1 + continue + + if not source_exists and dest_exists: + # Recovery case: file already moved, just update DB to match destination + logger.info(f'Reorganize recovery: updating DB for already-moved file: {dest_path}') + + dest_stem, _ = split_path_stem_and_suffix(dest_path) + + # Find existing files at destination that match the FileGroup's file extensions + moved_files = [] + for file_info in fg_info['files']: + _, suffix = split_path_stem_and_suffix(pathlib.Path(file_info['path'])) + dest_file = dest_path.parent / f'{dest_stem}{suffix}' + if dest_file.exists(): + # Map original source path to destination for DB update + src_file = fg_info['directory'] / file_info['path'] + moved_files.append((src_file, dest_file)) + + if moved_files: + # Build DB update using existing destination files + moved_lookup = {src: dst for src, dst in moved_files} + new_files = [] + for file_info in fg_info['files']: + old_file_path = pathlib.Path(file_info['path']) + if not old_file_path.is_absolute(): + old_file_path = fg_info['directory'] / old_file_path + if old_file_path in moved_lookup: + file_info = dict(file_info) + file_info['path'] = moved_lookup[old_file_path].name + new_files.append(file_info) + + new_data = fg_info['data'] + if new_data: + new_data = dict(new_data) + filename_lookup = {src.name: dst.name for src, dst in moved_files} + for key, value in new_data.items(): + if isinstance(value, str) and value in filename_lookup: + new_data[key] = filename_lookup[value] + elif isinstance(value, list): + new_data[key] = [filename_lookup.get(v, v) for v in value] + + pending_updates.append({ + 'id': fg_info['id'], + 'directory': str(dest_path.parent), + 'primary_path': str(dest_path), + 'files': new_files, + 'data': new_data, + }) + completed_moves.append((source_path, dest_path)) + + processed += 1 + if processed % PROGRESS_UPDATE_INTERVAL == 0: + percent = int((processed / total_moves) * 100) + self.update_status(operation_processed=processed, operation_percent=percent) + continue + + # Normal case: source exists, dest doesn't - move files + # Track source directory for cleanup + source_directories.add(source_path.parent) + + # Create destination directory if needed + if not dest_path.parent.exists(): + dest_path.parent.mkdir(parents=True, exist_ok=True) + created_directories.add(dest_path.parent) + + # Phase 2: Use FileGroup.files instead of glob_shared_stem() + source_stem_files = [] + for file_info in fg_info['files']: + file_path = pathlib.Path(file_info['path']) + if not file_path.is_absolute(): + file_path = fg_info['directory'] / file_path + source_stem_files.append(file_path) + + # Move all files in the FileGroup + dest_stem, _ = split_path_stem_and_suffix(dest_path) + moved_files = [] + for src_file in source_stem_files: + if not src_file.exists(): + continue + # Compute destination for this file based on its suffix + _, src_suffix = split_path_stem_and_suffix(src_file) + dest_file = dest_path.parent / f'{dest_stem}{src_suffix}' + + try: + shutil.move(str(src_file), str(dest_file)) + moved_files.append((src_file, dest_file)) + logger.debug(f'Reorganize: moved {src_file} -> {dest_file}') + except Exception as e: + logger.error(f'Failed to move {src_file} -> {dest_file}: {e}') + raise + + # Prepare FileGroup update + if moved_files: + # Phase 3: Use dict lookup instead of nested loop + moved_lookup = {src: dst for src, dst in moved_files} + + # Update files list with new paths + new_files = [] + for file_info in fg_info['files']: + old_file_path = pathlib.Path(file_info['path']) + if not old_file_path.is_absolute(): + old_file_path = fg_info['directory'] / old_file_path + + # Find the corresponding new path using dict lookup + if old_file_path in moved_lookup: + file_info = dict(file_info) + file_info['path'] = moved_lookup[old_file_path].name + new_files.append(file_info) + + # Update data dict if it has path references + # Data fields store filenames only (no '/'), so use filename-to-filename lookup + new_data = fg_info['data'] + if new_data: + new_data = dict(new_data) + # Build filename-to-filename lookup (not full paths) + filename_lookup = {src.name: dst.name for src, dst in moved_files} + for key, value in new_data.items(): + if isinstance(value, str) and value in filename_lookup: + new_data[key] = filename_lookup[value] + elif isinstance(value, list): + # Handle list fields like caption_paths + new_data[key] = [filename_lookup.get(v, v) for v in value] + + # Phase 4: Collect update for batch processing + pending_updates.append({ + 'id': fg_info['id'], + 'directory': str(dest_path.parent), + 'primary_path': str(dest_path), + 'files': new_files, + 'data': new_data, + }) + completed_moves.append((source_path, dest_path)) + + # Flush every MOVE_CHUNK_SIZE items + if len(pending_updates) >= MOVE_CHUNK_SIZE: + _bulk_update_file_groups_reorganize(pending_updates) + pending_updates = [] + + processed += 1 + + # Update progress + if processed % PROGRESS_UPDATE_INTERVAL == 0 or processed == total_moves: + percent = int((processed / total_moves) * 100) + logger.info(f'Reorganize progress: {processed}/{total_moves} files ({percent}%)') + self.update_status( + operation_processed=processed, + operation_percent=percent, + ) + + # Final flush of pending updates + if pending_updates: + _bulk_update_file_groups_reorganize(pending_updates) + + # Clean up empty source directories + for directory in sorted(source_directories, key=lambda p: len(p.parents), reverse=True): + try: + if directory.is_dir() and not any(directory.iterdir()): + delete_directory(directory) + logger.debug(f'Cleaned up empty directory: {directory}') + except (OSError, StopIteration): + pass + + # Skip post-processing - reorganization only moves files, nothing to re-index + + # Update file_format after successful completion (deferred update for resumability) + if task.collection_id and task.pending_file_format: + from wrolpi.collections.models import Collection + with get_db_session(commit=True) as session: + collection = session.query(Collection).filter_by(id=task.collection_id).one_or_none() + if collection: + collection.file_format = task.pending_file_format + logger.info(f'Updated file_format for collection {collection.name}') + # Trigger config save + if collection.kind == 'domain': + from modules.archive.lib import save_domains_config + save_domains_config.activate_switch() + elif collection.kind == 'channel': + from modules.videos.lib import save_channels_config + save_channels_config.activate_switch() + + logger.info(f'Reorganize completed: {len(completed_moves)} files moved') + Events.send_file_move_completed(f'Reorganized {len(completed_moves)} files') + self._complete_job(task.job_id) + + except asyncio.CancelledError: + logger.warning('Reorganize task was cancelled') + raise + except Exception as e: + logger.error(f'Reorganize failed: {e}', exc_info=e) + self.update_status(status='error', error=str(e)) + # Note: We don't attempt automatic rollback for reorganize tasks + # as the partial state may be complex. User should refresh files. + Events.send_file_move_failed(f'Reorganize failed: {e}') + finally: + self.reset_status() + + async def handle_batch_reorganize(self, task: FileTask): + """Handle batch reorganization of multiple collections. + + Processes collections sequentially: + 1. For each collection, execute reorganization + 2. Track overall progress and per-collection progress + 3. Stop on first failure, report which collection failed + + Progress tracking: + - overall_percent = (completed_collections + current_collection_%) / total_collections + - batch_status dict tracks completed list, current collection, and failure info + """ + from wrolpi.collections.reorganize import execute_reorganization + from wrolpi.collections.models import Collection + from wrolpi.events import Events + + collection_ids = task.collection_ids or [] + kind = task.batch_kind + + if not collection_ids: + logger.info('Batch reorganize task has no collections, nothing to do') + self._complete_job(task.job_id) + self.reset_status() + return + + total_collections = len(collection_ids) + logger.info(f'Starting batch reorganization of {total_collections} {kind} collections') + + # Transition job from 'pending' to 'running' + self._set_job_status(task.job_id, 'running') + + # Initialize batch status tracking + batch_status = { + 'batch_job_id': task.job_id, # Include job ID so frontend can resume polling + 'total_collections': total_collections, + 'completed': [], + 'current_collection': None, + 'failed_collection': None, + 'error': None, + 'overall_percent': 0, + } + + self.update_status( + status='batch_reorganizing', + task_type='batch_reorganize', + paths=[], + destination=None, + operation_total=total_collections, + operation_processed=0, + operation_percent=0, + error=None, + batch_status=batch_status, + collection_kind=kind, + ) + + try: + for idx, collection_id in enumerate(collection_ids): + # Fetch collection info + with get_db_session() as session: + collection = session.query(Collection).filter_by(id=collection_id).one_or_none() + if not collection: + logger.warning(f'Batch reorganize: collection {collection_id} not found, skipping') + continue + + collection_name = collection.name + + # Update current collection in status + batch_status['current_collection'] = { + 'id': collection_id, + 'name': collection_name, + 'status': 'running', + 'total': 0, + 'completed': 0, + 'percent': 0, + } + self.update_status(batch_status=batch_status) + + logger.info(f'Batch reorganize: processing {collection_name} ({idx + 1}/{total_collections})') + + try: + # Execute reorganization for this collection + # This queues a sub-job that we need to wait for + job_id = execute_reorganization(collection_id, session) + + if job_id: + # Wait for the reorganization job to complete + # Poll for completion while updating batch status + await self._wait_for_sub_job_with_progress( + job_id, batch_status, collection_id, collection_name, idx, total_collections + ) + + # Mark this collection as completed + batch_status['completed'].append({ + 'id': collection_id, + 'name': collection_name, + 'status': 'complete', + }) + + # Update overall progress + completed_count = len(batch_status['completed']) + batch_status['overall_percent'] = int((completed_count / total_collections) * 100) + batch_status['current_collection'] = None + self.update_status( + operation_processed=completed_count, + operation_percent=batch_status['overall_percent'], + batch_status=batch_status, + ) + + logger.info(f'Batch reorganize: completed {collection_name}') + + except Exception as e: + # Collection failed - stop batch processing + logger.error(f'Batch reorganize: {collection_name} failed: {e}') + batch_status['failed_collection'] = { + 'id': collection_id, + 'name': collection_name, + } + batch_status['error'] = str(e) + batch_status['current_collection'] = None + self.update_status( + status='error', + error=str(e), + batch_status=batch_status, + ) + self._set_job_status(task.job_id, 'failed') + Events.send_file_move_failed( + f'Batch reorganization failed on {collection_name}: {e}' + ) + return + + # All collections completed successfully + logger.info(f'Batch reorganization completed: {len(batch_status["completed"])} collections processed') + Events.send_file_move_completed( + f'Batch reorganization completed: {len(batch_status["completed"])} {kind}s reorganized' + ) + self._complete_job(task.job_id) + + except asyncio.CancelledError: + logger.warning('Batch reorganize task was cancelled') + raise + except Exception as e: + logger.error(f'Batch reorganize failed: {e}', exc_info=e) + batch_status['error'] = str(e) + self.update_status(status='error', error=str(e), batch_status=batch_status) + self._set_job_status(task.job_id, 'failed') + Events.send_file_move_failed(f'Batch reorganization failed: {e}') + finally: + self.reset_status() + + async def _wait_for_sub_job_with_progress( + self, + job_id: str, + batch_status: dict, + collection_id: int, + collection_name: str, + collection_idx: int, + total_collections: int, + ): + """Wait for a sub-job to complete while updating batch progress. + + This is used by handle_batch_reorganize to wait for each collection's + reorganization job while keeping the batch status updated. + """ + import time + start = time.time() + timeout = 3600 # 1 hour max per collection + + while time.time() - start < timeout: + status = self.get_job_status(job_id) + if status == 'complete': + return + + # Transfer and process queue FIRST to ensure status is updated + self.transfer_queue() + await self.process_queue() + + # Update current collection progress based on worker status (now fresh) + worker_status = self.status + current = batch_status.get('current_collection', {}) + if current: + current['total'] = worker_status.get('operation_total', 0) + current['completed'] = worker_status.get('operation_processed', 0) + current['percent'] = worker_status.get('operation_percent', 0) + batch_status['current_collection'] = current + + # Calculate overall percent including current progress + completed_count = len(batch_status.get('completed', [])) + current_progress = current['percent'] / 100 + batch_status['overall_percent'] = int( + ((completed_count + current_progress) / total_collections) * 100 + ) + self.update_status(batch_status=batch_status) + + await asyncio.sleep(0.05) + + raise TimeoutError(f'Collection {collection_name} reorganization timed out') + # All processes create a FileWorker, but only one receives the signals and actually does work. file_worker = FileWorker() diff --git a/wrolpi/media_path.py b/wrolpi/media_path.py index 8c492677..651a1e3c 100644 --- a/wrolpi/media_path.py +++ b/wrolpi/media_path.py @@ -18,7 +18,11 @@ class MediaPathType(types.TypeDecorator): # noqa raise ValueError('MediaPath cannot be empty') if isinstance(value, pathlib.Path): - value = str(value.absolute()) + if not value.is_absolute(): + # Resolve relative paths against media directory, not cwd + from wrolpi.common import get_media_directory + value = get_media_directory() / value + value = str(value) if not isinstance(value, str): raise ValueError(f'Invalid MediaPath type ({type(value)}): {value}') diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 7d25e4fe..16db46ba 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1015,29 +1015,60 @@ async def test_config_valid(async_client, test_wrolpi_config): @pytest.mark.parametrize('name,expected_name', [ ('foo', 'foo'), (pathlib.Path('foo'), pathlib.Path('foo')), + # Short filenames with parents (no trimming needed) should preserve parent directories. + ('some/parent/short.txt', 'some/parent/short.txt'), + ('/absolute/parent/short.txt', '/absolute/parent/short.txt'), + (pathlib.Path('/some/parent/short.txt'), pathlib.Path('/some/parent/short.txt')), + # Long ASCII filenames are trimmed to 200 BYTES (not characters) ( 'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.txt', - 'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt' + 'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt' ), ( 'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json', - 'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json' + 'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json' ), ( '/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json', - '/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json' + '/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json' ), ( pathlib.Path( '/some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.readability.json'), pathlib.Path( - '/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json') + '/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json') ), ]) def test_trim_file_name(name, expected_name): assert common.trim_file_name(name) == expected_name +def test_trim_file_name_utf8_bytes(): + """UTF-8 multi-byte characters must be trimmed by BYTE length, not character length. + + Linux ext4/xfs filesystems have a 255 BYTE limit, not a 255 character limit. + Hindi/Devanagari characters are 3 bytes each in UTF-8, so a 121-character filename + can be 263 bytes, exceeding the limit. + + This test reproduces the bug from: https://decay.local reorganization failure + """ + # This filename has 133 characters but 291 bytes - exceeds 255 byte limit + hindi_filename = '2022-04-13-19-40-51_भारतीय संस्कृति और परंपराएं बहुत समृद्ध हैं हमारे देश में विविधता में एकता है यह हमारी विरासत है.readability.html' + assert len(hindi_filename) == 133 # Characters + assert len(hindi_filename.encode('utf-8')) == 291 # Bytes - exceeds 255! + + result = common.trim_file_name(hindi_filename) + + # Result must fit within filesystem byte limit (200 bytes leaves room for suffix variations) + max_bytes = 200 + result_bytes = len(result.encode('utf-8')) + assert result_bytes <= max_bytes, \ + f'Trimmed filename is {result_bytes} bytes, exceeds {max_bytes} byte limit' + + # Must preserve the suffix + assert result.endswith('.readability.html'), f'Suffix not preserved: {result}' + + async def test_cached_multiprocessing_result(async_client): call_count = 0 diff --git a/wrolpi/test_media_path.py b/wrolpi/test_media_path.py new file mode 100644 index 00000000..44a28647 --- /dev/null +++ b/wrolpi/test_media_path.py @@ -0,0 +1,85 @@ +""" +Tests for MediaPathType to ensure paths are resolved correctly. +""" +import os +import pathlib + +import pytest + +from wrolpi.collections.models import Collection +from wrolpi.common import get_media_directory +from wrolpi.media_path import MediaPathType + + +def test_media_path_type_relative_path_resolved_against_media_directory(test_session, test_directory): + """MediaPathType should resolve relative paths against media directory, not cwd.""" + relative_path = pathlib.Path('archive/test.com') + collection = Collection(name='test.com', kind='domain', directory=relative_path) + test_session.add(collection) + test_session.commit() + test_session.expire(collection) + + assert collection.directory.is_absolute() + assert collection.directory == test_directory / 'archive/test.com' + + +def test_media_path_type_process_bind_param_relative_path(test_directory): + """process_bind_param should resolve relative paths against media directory.""" + media_path_type = MediaPathType() + relative_path = pathlib.Path('archive/test.com') + result = media_path_type.process_bind_param(relative_path, dialect=None) + + assert result == str(test_directory / 'archive/test.com') + assert not result.startswith(os.getcwd()) # NOT resolved against cwd + + +def test_collection_get_or_set_directory_database_roundtrip(test_session, test_directory, test_wrolpi_config): + """get_or_set_directory should store and retrieve correct paths.""" + from unittest import mock + + collection = Collection(name='example.com', kind='domain') + test_session.add(collection) + test_session.commit() + + # Mock switch activation since shared context isn't set up in unit tests + with mock.patch('modules.archive.lib.save_domains_config.activate_switch'): + result1 = collection.get_or_set_directory(test_session) + test_session.expire(collection) + result2 = collection.get_or_set_directory(test_session) + + assert result1 == result2 + assert str(test_directory) in str(result2) + assert '/opt/wrolpi' not in str(collection.directory) + + +def test_media_path_type_absolute_path_unchanged(test_session, test_directory): + """Absolute paths under media directory should be stored unchanged.""" + absolute_path = test_directory / 'archive/test.com' + collection = Collection(name='test.com', kind='domain', directory=absolute_path) + test_session.add(collection) + test_session.commit() + test_session.expire(collection) + + assert collection.directory == absolute_path + + +def test_media_path_type_none_unchanged(test_directory): + """None should pass through unchanged.""" + media_path_type = MediaPathType() + result = media_path_type.process_bind_param(None, dialect=None) + assert result is None + + +def test_media_path_type_empty_string_raises(test_directory): + """Empty string should raise ValueError.""" + media_path_type = MediaPathType() + with pytest.raises(ValueError, match='MediaPath cannot be empty'): + media_path_type.process_bind_param('', dialect=None) + + +def test_media_path_type_string_path_passed_through(test_directory): + """String paths should be passed through as-is (assumed already absolute).""" + media_path_type = MediaPathType() + path_str = str(test_directory / 'archive/test.com') + result = media_path_type.process_bind_param(path_str, dialect=None) + assert result == path_str