FileGroups in Collections can be reorganized in the Collection's directory.
Some checks failed
Controller Tests / controller-tests (push) Has been cancelled
Jest Tests / jest-tests (push) Has been cancelled

* Batch reorganization is supported for both Domain and Channel Collections.
* Batch reorganization happens when any Collection does not match the respective file format. For example, a Channel Collection file format that does not match the `yt_dlp_options.file_name_format` will show `needs_reorganization`.
* Individual Collections can always be reorganized. The reorganization is a little more aggressive. Useful for when some files may have been missed in the batch reorganization.
This commit is contained in:
Roland 2026-01-23 19:51:02 -07:00
parent 208ac4fac7
commit 6891ebeb62
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
49 changed files with 6614 additions and 181 deletions

View file

@ -0,0 +1,48 @@
"""add_collection_file_format
Revision ID: a1b2c3d4e5f6
Revises: 76add63ed225
Create Date: 2026-01-23
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'a1b2c3d4e5f6'
down_revision = '76add63ed225'
branch_labels = None
depends_on = None
def upgrade():
# Add file_format column to collection table
# This stores the file_name_format used when files were last organized
op.add_column('collection', sa.Column('file_format', sa.String(), nullable=True))
# Set default values from current configs so existing collections don't need reorganization
from modules.archive.lib import get_archive_downloader_config
from modules.videos.lib import get_videos_downloader_config
connection = op.get_bind()
# Update domain collections with archive config's file_name_format
archive_config = get_archive_downloader_config()
if archive_config.file_name_format:
connection.execute(
sa.text("UPDATE collection SET file_format = :format WHERE kind = 'domain'"),
{'format': archive_config.file_name_format}
)
# Update channel collections with video config's file_name_format
video_config = get_videos_downloader_config()
if video_config.file_name_format:
connection.execute(
sa.text("UPDATE collection SET file_format = :format WHERE kind = 'channel'"),
{'format': video_config.file_name_format}
)
def downgrade():
op.drop_column('collection', 'file_format')

View file

@ -0,0 +1,189 @@
"""Fix media paths incorrectly resolved against /opt/wrolpi.
MediaPathType.process_bind_param() was calling .absolute() on relative paths,
which resolves against the current working directory (cwd) instead of the
media directory. In production, cwd is /opt/wrolpi, so paths like
'archive/spiegel.de' became '/opt/wrolpi/archive/spiegel.de' instead of
'/media/wrolpi/archive/spiegel.de'.
This migration fixes any corrupted paths in the database and saves the
corrected paths to config files.
Revision ID: b2b9700f5048
Revises: a1b2c3d4e5f6
Create Date: 2026-01-27
"""
import os
import pathlib
import yaml
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'b2b9700f5048'
down_revision = 'a1b2c3d4e5f6'
branch_labels = None
depends_on = None
def get_relative_path(absolute_path: str, media_directory: str) -> str:
"""Convert absolute path to relative path under media directory."""
if absolute_path and absolute_path.startswith(media_directory):
return absolute_path[len(media_directory):].lstrip('/')
return absolute_path
def save_config_file(config: dict, file_path: pathlib.Path, width: int = 120):
"""Write config dict to YAML file."""
file_path.parent.mkdir(parents=True, exist_ok=True)
with file_path.open('wt') as fh:
yaml.dump(config, fh, width=width, sort_keys=True)
fh.flush()
os.fsync(fh.fileno())
def upgrade():
connection = op.get_bind()
# Fix collection.directory
result = connection.execute(sa.text("""
UPDATE collection
SET directory = REPLACE(directory, '/opt/wrolpi/', '/media/wrolpi/')
WHERE directory LIKE '/opt/wrolpi/%'
"""))
if result.rowcount:
print(f"Fixed {result.rowcount} collection directory paths")
# Fix file_group.directory
result = connection.execute(sa.text("""
UPDATE file_group
SET directory = REPLACE(directory, '/opt/wrolpi/', '/media/wrolpi/')
WHERE directory LIKE '/opt/wrolpi/%'
"""))
if result.rowcount:
print(f"Fixed {result.rowcount} file_group directory paths")
# Fix file_group.primary_path
result = connection.execute(sa.text("""
UPDATE file_group
SET primary_path = REPLACE(primary_path, '/opt/wrolpi/', '/media/wrolpi/')
WHERE primary_path LIKE '/opt/wrolpi/%'
"""))
if result.rowcount:
print(f"Fixed {result.rowcount} file_group primary_path paths")
# Fix download.destination
result = connection.execute(sa.text("""
UPDATE download
SET destination = REPLACE(destination, '/opt/wrolpi/', '/media/wrolpi/')
WHERE destination LIKE '/opt/wrolpi/%'
"""))
if result.rowcount:
print(f"Fixed {result.rowcount} download destination paths")
# Save corrected paths to config files directly (bypass config framework which needs SharedContext)
try:
from wrolpi.common import get_media_directory
media_dir = str(get_media_directory())
config_dir = get_media_directory() / 'config'
# Save domains config
domains_result = connection.execute(sa.text("""
SELECT id, name, kind, description, directory, file_format
FROM collection
WHERE kind = 'domain'
ORDER BY name
"""))
domains_collections = []
for row in domains_result:
collection_config = {'name': row[1], 'kind': row[2]}
if row[3]: # description
collection_config['description'] = row[3]
if row[4]: # directory
collection_config['directory'] = get_relative_path(row[4], media_dir + '/')
if row[5]: # file_format
collection_config['file_format'] = row[5]
# Get tag name if exists
tag_result = connection.execute(sa.text("""
SELECT t.name FROM tag t
JOIN collection c ON c.tag_id = t.id
WHERE c.id = :id
"""), {'id': row[0]}).first()
collection_config['tag_name'] = tag_result[0] if tag_result else None
domains_collections.append(collection_config)
# Read existing config to preserve version
domains_file = config_dir / 'domains.yaml'
version = 0
if domains_file.exists():
with domains_file.open() as f:
existing = yaml.safe_load(f) or {}
version = existing.get('version', 0)
domains_config = {'version': version + 1, 'collections': domains_collections}
save_config_file(domains_config, domains_file)
print(f"Saved domains config with {len(domains_collections)} collections")
# Save channels config
# Columns: 0=id, 1=name, 2=kind, 3=description, 4=directory, 5=file_format,
# 6=url, 7=source_id, 8=generate_posters, 9=calculate_duration, 10=download_missing_data
channels_result = connection.execute(sa.text("""
SELECT c.id, c.name, c.kind, c.description, c.directory, c.file_format,
ch.url, ch.source_id, ch.generate_posters, ch.calculate_duration,
ch.download_missing_data
FROM collection c
JOIN channel ch ON ch.collection_id = c.id
WHERE c.kind = 'channel'
ORDER BY c.name
"""))
channels_list = []
for row in channels_result:
channel_config = {'name': row[1]}
if row[4]: # directory
channel_config['directory'] = get_relative_path(row[4], media_dir + '/')
if row[6]: # url
channel_config['url'] = row[6]
if row[7]: # source_id
channel_config['source_id'] = row[7]
channel_config['generate_posters'] = row[8]
channel_config['calculate_duration'] = row[9]
channel_config['download_missing_data'] = row[10]
if row[5]: # file_format
channel_config['file_format'] = row[5]
# Get tag name if exists
tag_result = connection.execute(sa.text("""
SELECT t.name FROM tag t
JOIN collection c ON c.tag_id = t.id
WHERE c.id = :id
"""), {'id': row[0]}).first()
channel_config['tag_name'] = tag_result[0] if tag_result else None
# Get downloads for this channel's collection
downloads_result = connection.execute(sa.text("""
SELECT url, frequency FROM download
WHERE collection_id = :collection_id
"""), {'collection_id': row[0]})
downloads = [{'url': d[0], 'frequency': d[1]} for d in downloads_result]
if downloads:
channel_config['downloads'] = downloads
channels_list.append(channel_config)
# Read existing config to preserve version
channels_file = config_dir / 'channels.yaml'
version = 0
if channels_file.exists():
with channels_file.open() as f:
existing = yaml.safe_load(f) or {}
version = existing.get('version', 0)
channels_config = {'version': version + 1, 'channels': channels_list}
save_config_file(channels_config, channels_file)
print(f"Saved channels config with {len(channels_list)} channels")
except Exception as e:
print(f"Warning: Could not save configs: {e}")
def downgrade():
# Data fix, no meaningful downgrade
pass

View file

@ -11,6 +11,7 @@ import {InventoryRoute} from "./components/Inventory";
import {ArchiveRoute} from "./components/Archive";
import {FilesRoute} from "./components/Files";
import {QueryProvider, StatusProvider} from "./hooks/customHooks";
import {FileWorkerStatusProvider} from "./contexts/FileWorkerStatusContext";
import {MapRoute} from "./components/Map";
import {MediaContextProvider, mediaStyles, StatusContext, ThemeContext} from "./contexts/contexts";
import {Header, ThemeProvider} from "./components/Theme";
@ -98,12 +99,14 @@ export default function App() {
useEventsInterval();
return <StatusProvider>
{/* Context and style to handle switching between mobile/computer. */}
<style>{mediaStyles}</style>
{/* Toasts can be on any page. */}
<SemanticToastContainer position='top-right'/>
<MediaContextProvider>
<RouterProvider router={router}/>
</MediaContextProvider>
<FileWorkerStatusProvider>
{/* Context and style to handle switching between mobile/computer. */}
<style>{mediaStyles}</style>
{/* Toasts can be on any page. */}
<SemanticToastContainer position='top-right'/>
<MediaContextProvider>
<RouterProvider router={router}/>
</MediaContextProvider>
</FileWorkerStatusProvider>
</StatusProvider>
}

View file

@ -442,6 +442,14 @@ export async function postArchiveFileFormat(archive_file_format) {
return await apiPost(`${ARCHIVES_API}/file_format`, body);
}
export async function fetchArchiveBrowsers() {
const response = await apiGet(`${ARCHIVES_API}/browsers`);
if (response.ok) {
return await response.json();
}
return {browsers: [], available: false};
}
export async function fetchBrowserProfiles() {
const response = await apiGet(`${API_URI}/videos/browser-profiles`);
if (response.status === 200) {
@ -888,6 +896,126 @@ export async function refreshDomain(domainId) {
return response;
}
export async function previewCollectionReorganization(collectionId) {
const response = await apiGet(`${COLLECTIONS_API}/${collectionId}/reorganize/preview`);
if (response.ok) {
return await response.json();
} else {
const message = await getErrorMessage(response, 'Unable to preview reorganization. See server logs.');
toast({
type: 'error',
title: 'Reorganization Preview Error',
description: message,
time: 5000,
});
throw new Error(message);
}
}
export async function executeCollectionReorganization(collectionId) {
const response = await apiPost(`${COLLECTIONS_API}/${collectionId}/reorganize`);
if (response.ok) {
const data = await response.json();
if (data.job_id) {
toast({
type: 'success',
title: 'Reorganization Started',
description: 'Files are being reorganized in the background',
time: 3000,
});
} else {
toast({
type: 'info',
title: 'No Changes Needed',
description: data.message || 'No files need reorganization',
time: 3000,
});
}
return data;
} else {
const message = await getErrorMessage(response, 'Unable to start reorganization. See server logs.');
toast({
type: 'error',
title: 'Reorganization Error',
description: message,
time: 5000,
});
throw new Error(message);
}
}
export async function getReorganizationStatus(collectionId, jobId) {
const response = await apiGet(`${COLLECTIONS_API}/${collectionId}/reorganize/status/${jobId}`);
if (response.ok) {
return await response.json();
} else {
const message = await getErrorMessage(response, 'Unable to get reorganization status.');
throw new Error(message);
}
}
// Batch Reorganization API Functions
export async function previewBatchReorganization(kind) {
const endpoint = kind === 'channel' ? 'channels' : 'domains';
const response = await apiGet(`${COLLECTIONS_API}/reorganize/${endpoint}`);
if (response.ok) {
return await response.json();
} else {
const message = await getErrorMessage(response, `Unable to get ${kind}s needing reorganization.`);
toast({
type: 'error',
title: 'Batch Reorganization Preview Error',
description: message,
time: 5000,
});
throw new Error(message);
}
}
export async function executeBatchReorganization(kind) {
const endpoint = kind === 'channel' ? 'channels' : 'domains';
const response = await apiPost(`${COLLECTIONS_API}/reorganize/${endpoint}`);
if (response.ok) {
const data = await response.json();
if (data.batch_job_id) {
toast({
type: 'success',
title: 'Batch Reorganization Started',
description: `${data.collection_count} ${kind}s are being reorganized`,
time: 3000,
});
} else {
toast({
type: 'info',
title: 'No Changes Needed',
description: data.message || `No ${kind}s need reorganization`,
time: 3000,
});
}
return data;
} else {
const message = await getErrorMessage(response, `Unable to start batch reorganization. See server logs.`);
toast({
type: 'error',
title: 'Batch Reorganization Error',
description: message,
time: 5000,
});
throw new Error(message);
}
}
export async function getBatchReorganizationStatus(batchJobId) {
const response = await apiGet(`${COLLECTIONS_API}/reorganize/batch/status/${batchJobId}`);
if (response.ok) {
return await response.json();
} else {
const message = await getErrorMessage(response, 'Unable to get batch reorganization status.');
throw new Error(message);
}
}
export async function getArchive(archiveId) {
const response = await apiGet(`${ARCHIVES_API}/${archiveId}`);
if (response.ok) {

View file

@ -44,11 +44,13 @@ import {
import {
deleteArchives,
deleteDomain,
fetchArchiveBrowsers,
fetchArchiveDownloaderConfig,
generateArchiveScreenshot,
getCollectionTagInfo,
postArchiveFileFormat,
postDownload,
previewBatchReorganization,
refreshDomain,
tagDomain,
tagFileGroup,
@ -57,10 +59,13 @@ import {
} from "../api";
import {InputForm, useForm} from "../hooks/useForm";
import {CollectionTagModal} from "./collections/CollectionTagModal";
import {CollectionReorganizeModal} from "./collections/CollectionReorganizeModal";
import {BatchReorganizeModal} from "./collections/BatchReorganizeModal";
import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {
useArchive,
useDockerized,
useDomain,
useDomains,
useOneQuery,
@ -505,6 +510,8 @@ export function DomainEditPage() {
// Modal state for tagging
const [tagEditModalOpen, setTagEditModalOpen] = useState(false);
// Modal state for reorganization
const [reorganizeModalOpen, setReorganizeModalOpen] = useState(false);
useTitle(`Edit Domain: ${domain?.domain || '...'}`);
@ -612,10 +619,21 @@ export function DomainEditPage() {
style={{marginTop: '1em'}}
>Tag</Button>;
const reorganizeButton = (
<APIButton
color='orange'
size='small'
onClick={() => setReorganizeModalOpen(true)}
obeyWROLMode={true}
style={{marginTop: '1em'}}
>Reorganize Files</APIButton>
);
const actionButtons = <>
{deleteButton}
{refreshButton}
{tagButton}
{reorganizeButton}
</>;
const [descriptionProps] = form.getCustomProps({name: 'description', path: 'description'});
@ -626,6 +644,16 @@ export function DomainEditPage() {
<Button>Archives</Button>
</Link>
{domain?.needs_reorganization && (
<Message warning>
<Message.Header>File Format Changed</Message.Header>
<p>
The file name format has changed. Click "Reorganize Files" to move existing files
to match the new format.
</p>
</Message>
)}
<CollectionEditForm
form={form}
title={`Edit Domain: ${domain?.domain || '...'}`}
@ -670,6 +698,16 @@ export function DomainEditPage() {
hasDirectory={!!domain?.directory}
/>
{/* Reorganize Modal */}
<CollectionReorganizeModal
open={reorganizeModalOpen}
onClose={() => setReorganizeModalOpen(false)}
collectionId={domain?.id}
collectionName={domain?.domain}
onComplete={fetchDomain}
needsReorganization={domain?.needs_reorganization}
/>
{/* Downloads Segment */}
<Segment>
<Header as='h1'>Downloads</Header>
@ -726,13 +764,145 @@ function ArchiveFileNameForm({form}) {
/>
}
function BrowserConfigForm({form, browsers, browsersAvailable}) {
const {t} = React.useContext(ThemeContext);
const [useCustomPath, setUseCustomPath] = useState(false);
// Determine if custom path is being used based on current value
React.useEffect(() => {
const currentBrowser = form.formData?.browser_executable;
if (currentBrowser) {
// Check if current value matches any known browser
const isKnownBrowser = browsers.some(b => b.path === currentBrowser || b.key === currentBrowser);
setUseCustomPath(!isKnownBrowser);
}
}, [form.formData?.browser_executable, browsers]);
if (!browsersAvailable) {
return null;
}
// Build dropdown options: auto-detect + installed browsers + custom
const browserOptions = [
{key: 'auto', value: '', text: 'Auto-detect (recommended)'},
...browsers.map(b => ({key: b.key, value: b.path, text: `${b.name} (${b.path})`})),
{key: 'custom', value: '__custom__', text: 'Custom path...'},
];
const handleBrowserChange = (e, {value}) => {
if (value === '__custom__') {
setUseCustomPath(true);
form.setValue('browser_executable', '');
} else {
setUseCustomPath(false);
form.setValue('browser_executable', value || null);
}
};
const currentValue = form.formData?.browser_executable || '';
const dropdownValue = useCustomPath ? '__custom__' : currentValue;
return <>
<Header as='h4'>Browser Settings</Header>
<p {...t}>
Configure which browser SingleFile uses to create archives.
These settings only apply to native deployments (Raspberry Pi/Debian).
</p>
<Form.Field>
<label {...t}>Browser</label>
<Form.Dropdown
selection
options={browserOptions}
value={dropdownValue}
onChange={handleBrowserChange}
placeholder='Select browser...'
/>
</Form.Field>
{useCustomPath && (
<Form.Field>
<label {...t}>Custom Browser Path</label>
<Input
fluid
placeholder='/usr/bin/chromium'
value={form.formData?.browser_executable || ''}
onChange={(e) => form.setValue('browser_executable', e.target.value)}
/>
<small {...t}>Enter the absolute path to the browser executable.</small>
</Form.Field>
)}
<Form.Field>
<label {...t}>Browser Arguments</label>
<Input
fluid
placeholder='["--no-sandbox"]'
value={form.formData?.browser_args || '["--no-sandbox"]'}
onChange={(e) => form.setValue('browser_args', e.target.value)}
/>
<small {...t}>JSON array of arguments passed to the browser. Default: ["--no-sandbox"]</small>
</Form.Field>
<Form.Field>
<label {...t}>User Agent</label>
<Input
fluid
placeholder='Leave empty to use system default'
value={form.formData?.user_agent || ''}
onChange={(e) => form.setValue('user_agent', e.target.value || null)}
/>
<small {...t}>Custom user agent string. Leave empty to use the system default.</small>
</Form.Field>
</>;
}
function ArchiveSettingsPage() {
useTitle('Archive Settings');
const {t} = React.useContext(ThemeContext);
const dockerized = useDockerized();
const [batchModalOpen, setBatchModalOpen] = useState(false);
const [domainsNeedingReorg, setDomainsNeedingReorg] = useState(0);
const [fetchingReorgCount, setFetchingReorgCount] = useState(true);
const [browsers, setBrowsers] = useState([]);
const [browsersAvailable, setBrowsersAvailable] = useState(false);
// Check how many domains need reorganization on mount
React.useEffect(() => {
setFetchingReorgCount(true);
previewBatchReorganization('domain')
.then(data => {
setDomainsNeedingReorg(data.total_collections || 0);
})
.catch(() => {
setDomainsNeedingReorg(0);
})
.finally(() => {
setFetchingReorgCount(false);
});
}, []);
// Fetch available browsers on mount (only if not dockerized)
React.useEffect(() => {
if (dockerized === false) {
fetchArchiveBrowsers()
.then(data => {
setBrowsers(data.browsers || []);
setBrowsersAvailable(data.available === true);
})
.catch(() => {
setBrowsers([]);
setBrowsersAvailable(false);
});
}
}, [dockerized]);
const emptyFormData = {
file_name_format: '%(download_datetime)s_%(title)s.%(ext)s',
browser_executable: null,
browser_args: '["--no-sandbox"]',
user_agent: null,
};
const configSubmitter = async () => {
@ -769,6 +939,20 @@ function ArchiveSettingsPage() {
<ArchiveFileNameForm form={configForm}/>
</GridColumn>
</GridRow>
{/* Browser settings - only shown on native deployments */}
{!dockerized && browsersAvailable && (
<GridRow columns={1}>
<GridColumn mobile={16} computer={8}>
<BrowserConfigForm
form={configForm}
browsers={browsers}
browsersAvailable={browsersAvailable}
/>
</GridColumn>
</GridRow>
)}
<GridRow columns={1}>
<GridColumn textAlign='right'>
<APIButton
@ -784,6 +968,40 @@ function ArchiveSettingsPage() {
</Form>
</Segment>
<Segment>
<Header as='h4'>File Organization</Header>
<p>
{fetchingReorgCount
? 'Checking for domains that need reorganization...'
: <>
<strong>{domainsNeedingReorg}</strong> domain{domainsNeedingReorg !== 1 ? 's' : ''}
{domainsNeedingReorg > 0
? ' have files that do not match the current file name format.'
: '. All domains are organized correctly.'}
</>
}
</p>
<Button
color='orange'
onClick={() => setBatchModalOpen(true)}
id='reorganize_all_domains_button'
disabled={fetchingReorgCount || domainsNeedingReorg === 0}
loading={fetchingReorgCount}
>
<Icon name='folder open outline'/> Reorganize All Domains
</Button>
</Segment>
<BatchReorganizeModal
open={batchModalOpen}
onClose={() => setBatchModalOpen(false)}
kind='domain'
onComplete={() => {
setBatchModalOpen(false);
setDomainsNeedingReorg(0);
}}
/>
<Segment>
<Header as='h3'>SingleFile Browser Extension</Header>

View file

@ -2,6 +2,7 @@ import React, {useState} from "react";
import {Grid, StatisticLabel, StatisticValue,} from "semantic-ui-react";
import {createChannel, deleteChannel, refreshChannel, tagChannel, tagChannelInfo} from "../api";
import {CollectionTagModal} from "./collections/CollectionTagModal";
import {CollectionReorganizeModal} from "./collections/CollectionReorganizeModal";
import {
APIButton,
BackButton,
@ -88,6 +89,7 @@ export function ChannelEditPage() {
const [tagEditModalOpen, setTagEditModalOpen] = useState(false);
const [downloadModalOpen, setDownloadModalOpen] = useState(false);
const [reorganizeModalOpen, setReorganizeModalOpen] = useState(false);
const {channel, form, fetchChannel} = useChannel(channelId);
@ -192,10 +194,21 @@ export function ChannelEditPage() {
style={{marginTop: '1em'}}
>Tag</Button>;
const reorganizeButton = (
<APIButton
color='orange'
size='small'
onClick={() => setReorganizeModalOpen(true)}
obeyWROLMode={true}
style={{marginTop: '1em'}}
>Reorganize Files</APIButton>
);
const actionButtons = <>
{deleteButton}
{refreshButton}
{tagButton}
{reorganizeButton}
</>;
const downloadMissingDataInfo = 'Automatically download missing comments, etc, in the background.';
@ -207,6 +220,16 @@ export function ChannelEditPage() {
<Button>Videos</Button>
</Link>
{channel?.needs_reorganization && (
<Message warning>
<Message.Header>File Format Changed</Message.Header>
<p>
The file name format has changed. Click "Reorganize Files" to move existing files
to match the new format.
</p>
</Message>
)}
<CollectionEditForm
form={form}
title="Edit Channel"
@ -279,6 +302,16 @@ export function ChannelEditPage() {
hasDirectory={!!channel.directory}
/>
{/* Reorganize Modal */}
<CollectionReorganizeModal
open={reorganizeModalOpen}
onClose={() => setReorganizeModalOpen(false)}
collectionId={channel?.collection_id}
collectionName={channel?.name}
onComplete={fetchChannel}
needsReorganization={channel?.needs_reorganization}
/>
{/* Downloads Segment */}
<Segment>
<Grid columns={2}>

View file

@ -780,7 +780,7 @@ export function InfoPopup({
position = 'left center',
iconSize = null,
header = '',
iconStyle = {marginLeft: '0.25em', marginRight: '0.25em', marginTop: '-0.5em'},
iconStyle = { margin: '0.5em'},
...props
}) {
const trigger = <Icon link name={icon} size={iconSize} style={iconStyle}/>;

View file

@ -54,6 +54,19 @@ jest.mock('./collections/CollectionTagModal', () => ({
},
}));
// Mock CollectionReorganizeModal
jest.mock('./collections/CollectionReorganizeModal', () => ({
CollectionReorganizeModal: ({open, onClose, collectionId, collectionName, onComplete}) => {
if (!open) return null;
return (
<div data-testid="collection-reorganize-modal">
<div data-testid="modal-header">Reorganize Files: {collectionName}</div>
<button data-testid="close-button" onClick={onClose}>Close</button>
</div>
);
},
}));
// Mock API functions
const mockGetCollectionTagInfo = jest.fn();
jest.mock('../api', () => ({

View file

@ -33,7 +33,6 @@ import {
useTitle
} from "./Common";
import {
useFilesProgressInterval,
usePages,
useSearchFiles,
useSearchFilter,
@ -41,6 +40,7 @@ import {
useStatusFlag,
useWROLMode
} from "../hooks/customHooks";
import {useFileWorkerStatus} from "../contexts/FileWorkerStatusContext";
import {Route, Routes} from "react-router";
import {CardPlaceholder} from "./Placeholder";
import {ArchiveCard, ArchiveRowCells} from "./Archive";
@ -566,7 +566,7 @@ function getPhaseLabel(status) {
}
export function FilesRefreshProgress() {
const {progress} = useFilesProgressInterval();
const {status: progress} = useFileWorkerStatus();
if (!progress || progress.status === 'idle') {
return null;

View file

@ -5,6 +5,7 @@ import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/c
import {DarkModeToggle, HotspotStatusIcon, useLocalStorage} from "./Common";
import {ShareButton} from "./Share";
import {useCPUTemperature, useIOStats, useLoad, useMemoryStats, usePowerStats, useWROLMode} from "../hooks/customHooks";
import {useReorganizationStatus} from "../contexts/FileWorkerStatusContext";
import {SearchIconButton} from "./Search";
import {Icon, Popup} from "./Theme";
import {HELP_VIEWER_URI, NAME} from "./Vars";
@ -165,10 +166,33 @@ export function NavBar() {
// issue.
const warningIcon = temperatureIcon || diskWaitIcon || memoryIcon || systemLoadIcon;
const {isReorganizing, taskType, collectionId, collectionKind} = useReorganizationStatus();
let processingLink;
if (status && status.flags) {
if (status.flags.file_worker_busy) {
processingLink = '/files';
// Smart navigation based on what the file worker is doing
if (isReorganizing && collectionId && collectionKind) {
// Single collection reorganization - navigate to that collection
if (collectionKind === 'channel') {
processingLink = `/videos/channel/${collectionId}/edit`;
} else if (collectionKind === 'domain') {
processingLink = `/archive/domain/${collectionId}/edit`;
} else {
processingLink = '/files';
}
} else if (isReorganizing && taskType === 'batch_reorganize' && collectionKind) {
// Batch reorganization - navigate to settings page for that kind
if (collectionKind === 'channel') {
processingLink = '/videos/settings';
} else if (collectionKind === 'domain') {
processingLink = '/archive/settings';
} else {
processingLink = '/files';
}
} else {
processingLink = '/files';
}
} else if (status.flags.map_importing) {
processingLink = '/map/manage';
}

View file

@ -57,6 +57,8 @@ import _ from "lodash";
import {defaultFileOrder, defaultSearchOrder, HELP_VIEWER_URI} from "./Vars";
import {InputForm, ToggleForm, useForm} from "../hooks/useForm";
import {VideoResolutionSelectorForm} from "./Download";
import {BatchReorganizeModal} from "./collections/BatchReorganizeModal";
import {previewBatchReorganization} from "../api";
export function VideoWrapper() {
const {videoId} = useParams();
@ -234,15 +236,25 @@ function VideosPage() {
function VideoFileNameForm({form}) {
const [message, setMessage] = React.useState(null);
const onChange = async (value) => {
const response = await postVideoFileFormat(value);
const {error, preview} = await response.json();
if (error) {
setMessage({content: error, header: 'Invalid File Name', negative: true});
} else {
setMessage({content: preview, header: 'File Name Preview', positive: true});
}
}
const debouncedOnChange = React.useCallback(
_.debounce(async (value) => {
const response = await postVideoFileFormat(value);
const {error, preview} = await response.json();
if (error) {
setMessage({content: error, header: 'Invalid File Name', negative: true});
} else {
setMessage({content: preview, header: 'File Name Preview', positive: true});
}
}, 300),
[]
);
// Cleanup debounce on unmount
React.useEffect(() => {
return () => debouncedOnChange.cancel();
}, [debouncedOnChange]);
const onChange = debouncedOnChange;
const label = <InfoHeader
headerSize='h5'
@ -257,10 +269,10 @@ function VideoFileNameForm({form}) {
<li><code>%(upload_year)s</code> - Upload year (YYYY)</li>
<li><code>%(upload_month)s</code> - Upload month (01-12)</li>
<li><code>%(id)s</code> - Video ID</li>
<li><code>%(ext)s</code> - File extension</li>
<li><code>%(channel)s</code> - Channel name</li>
<li><code>%(duration)s</code> - Duration in seconds</li>
<li><code>%(playlist_index)s</code> - Playlist index</li>
<li><code>%(ext)s</code> - File extension (required, must be at end)</li>
</ul>
<p>See yt-dlp docs for all variables. Subdirectories supported.</p>
</>}
@ -279,6 +291,25 @@ function VideoFileNameForm({form}) {
function VideosSettingsPage() {
useTitle('Videos Settings');
const [batchModalOpen, setBatchModalOpen] = useState(false);
const [channelsNeedingReorg, setChannelsNeedingReorg] = useState(0);
const [fetchingReorgCount, setFetchingReorgCount] = useState(true);
// Check how many channels need reorganization on mount
useEffect(() => {
setFetchingReorgCount(true);
previewBatchReorganization('channel')
.then(data => {
setChannelsNeedingReorg(data.total_collections || 0);
})
.catch(() => {
setChannelsNeedingReorg(0);
})
.finally(() => {
setFetchingReorgCount(false);
});
}, []);
const emptyFormData = {
video_resolutions: ['1080p', '720p', '480p', 'maximum'],
yt_dlp_options: {
@ -337,10 +368,11 @@ function VideosSettingsPage() {
popupContent='Always download videos with the selected browser profile. This is risky, and therefore discouraged.'
/>;
return <Segment>
<Header as='h3'>Video Downloader Config</Header>
return <>
<Segment>
<Header as='h3'>Video Downloader Config</Header>
<Form>
<Form>
<Grid>
<Grid.Row columns={2}>
<Grid.Column mobile={16} computer={8}>
@ -435,7 +467,42 @@ function VideosSettingsPage() {
</Grid.Row>
</Grid>
</Form>
</Segment>
</Segment>
<Segment>
<Header as='h4'>File Organization</Header>
<p>
{fetchingReorgCount
? 'Checking for channels that need reorganization...'
: <>
<strong>{channelsNeedingReorg}</strong> channel{channelsNeedingReorg !== 1 ? 's' : ''}
{channelsNeedingReorg > 0
? ' have files that do not match the current file name format.'
: '. All channels are organized correctly.'}
</>
}
</p>
<Button
color='orange'
onClick={() => setBatchModalOpen(true)}
id='reorganize_all_channels_button'
disabled={fetchingReorgCount || channelsNeedingReorg === 0}
loading={fetchingReorgCount}
>
<Icon name='folder open outline'/> Reorganize All Channels
</Button>
</Segment>
<BatchReorganizeModal
open={batchModalOpen}
onClose={() => setBatchModalOpen(false)}
kind='channel'
onComplete={() => {
setBatchModalOpen(false);
setChannelsNeedingReorg(0);
}}
/>
</>
}

View file

@ -303,15 +303,15 @@ export function StatusPage() {
{cpuProgress}
{memoryUsageProgress}
<Statistic.Group size='mini'>
<LoadStatistic label='1 Minute Load' value={minute_1} cores={cores}/>
<LoadStatistic label='5 Minute Load' value={minute_5} cores={cores}/>
<LoadStatistic label='15 Minute Load' value={minute_15} cores={cores}/>
<CPUTemperatureStatistic
id='cpu_temperature_statistic'
temperature={temperature}
high_temperature={high_temperature}
critical_temperature={critical_temperature}
/>
<LoadStatistic label='1 Minute Load' value={minute_1} cores={cores}/>
<LoadStatistic label='5 Minute Load' value={minute_5} cores={cores}/>
<LoadStatistic label='15 Minute Load' value={minute_15} cores={cores}/>
<IOWaitStatistic percent_iowait={percent_iowait}/>
<UptimeStatistic uptime_seconds={uptime_seconds}/>
</Statistic.Group>
@ -322,6 +322,9 @@ export function StatusPage() {
{cpuProgress}
{memoryUsageProgress}
<Statistic.Group>
<LoadStatistic label='1 Min. Load' value={minute_1} cores={cores}/>
<LoadStatistic label='5 Min.' value={minute_5} cores={cores}/>
<LoadStatistic label='15 Min.' value={minute_15} cores={cores}/>
<CPUTemperatureStatistic
id='cpu_temperature_statistic'
temperature={temperature}
@ -329,9 +332,6 @@ export function StatusPage() {
critical_temperature={critical_temperature}
style={{marginRight: 0}}
/>
<LoadStatistic label='1 Min. Load' value={minute_1} cores={cores}/>
<LoadStatistic label='5 Min.' value={minute_5} cores={cores}/>
<LoadStatistic label='15 Min.' value={minute_15} cores={cores}/>
<IOWaitStatistic percent_iowait={percent_iowait}/>
<UptimeStatistic uptime_seconds={uptime_seconds}/>
</Statistic.Group>

View file

@ -0,0 +1,411 @@
import React, {useCallback, useEffect, useState} from 'react';
import {Grid, Icon, Label, Progress} from 'semantic-ui-react';
import Message from 'semantic-ui-react/dist/commonjs/collections/Message';
import {Button, Modal, Table} from '../Theme';
import {APIButton} from '../Common';
import {
executeBatchReorganization,
getBatchReorganizationStatus,
previewBatchReorganization
} from '../../api';
import {useReorganizationStatus, useFileWorkerStatus} from '../../contexts/FileWorkerStatusContext';
/**
* Modal component for previewing and executing batch collection file reorganization.
*
* Shows a list of collections that need reorganization, allows user to execute
* batch reorganization, and tracks progress with dual progress bars (overall and per-collection).
*
* @param {boolean} open - Whether the modal is open
* @param {Function} onClose - Callback when modal is closed
* @param {string} kind - 'channel' or 'domain'
* @param {Function} onComplete - Optional callback when reorganization completes
*/
export function BatchReorganizeModal({
open,
onClose,
kind,
onComplete,
}) {
const [loading, setLoading] = useState(false);
const [preview, setPreview] = useState(null);
const [error, setError] = useState(null);
const [batchJobId, setBatchJobId] = useState(null);
const [status, setStatus] = useState(null);
const [polling, setPolling] = useState(false);
const kindLabel = kind === 'channel' ? 'Channel' : 'Domain';
const kindLabelPlural = kind === 'channel' ? 'Channels' : 'Domains';
// Check if batch reorganization is currently active
const {
isReorganizing,
taskType,
collectionKind,
batchStatus: workerBatchStatus,
workerStatus,
} = useReorganizationStatus();
const {setFastPolling} = useFileWorkerStatus();
// Enable fast polling when modal is open
useEffect(() => {
if (open) {
setFastPolling(true);
}
return () => setFastPolling(false);
}, [open, setFastPolling]);
// Load preview when modal opens, or resume if already reorganizing
useEffect(() => {
if (open && kind) {
// Check if batch_status has an active job for this kind
// This is the primary indicator since worker status can briefly show
// 'reorganizing'/'reorganize' during individual collection processing
const hasActiveBatchJob = workerBatchStatus?.batch_job_id &&
collectionKind === kind;
// Worker is actively doing something (taskType is set)
// After batch completes, taskType becomes null, so this distinguishes
// "batch in progress" from "batch completed but batch_job_id still cached"
const workerIsActive = taskType != null;
// Use either the standard detection OR the batch_status fallback (only when worker is active)
const isThisActive = (isReorganizing && taskType === 'batch_reorganize' && collectionKind === kind) ||
(hasActiveBatchJob && workerIsActive);
if (isThisActive && workerBatchStatus) {
// Resume: start polling API immediately for accurate progress
// Don't use workerBatchStatus.current_collection directly as it may have stale
// percent/total/completed values. The batch status API returns fresh values
// by copying operation_percent into current_collection.percent.
setLoading(false);
setPreview(null);
setBatchJobId(workerBatchStatus.batch_job_id || null);
// Don't setStatus from workerBatchStatus - let the first poll populate it
// with accurate values from the batch status API
setPolling(true);
} else if (!polling && !batchJobId) {
// Normal: fetch preview (only if not already polling/resuming)
setLoading(true);
setError(null);
setPreview(null);
setBatchJobId(null);
setStatus(null);
previewBatchReorganization(kind)
.then(data => {
setPreview(data);
setLoading(false);
})
.catch(err => {
setError(err.message);
setLoading(false);
});
}
}
}, [open, kind, isReorganizing, taskType, collectionKind]);
// Poll for status when we have a batch job ID
useEffect(() => {
if (!batchJobId || !polling) {
return;
}
let timeoutId = null;
let cancelled = false;
const pollStatus = async () => {
if (cancelled) {
return;
}
try {
const statusData = await getBatchReorganizationStatus(batchJobId);
if (cancelled) return;
setStatus(statusData);
if (statusData.status === 'complete') {
setPolling(false);
if (onComplete) {
onComplete();
}
} else if (statusData.status === 'failed' || statusData.error) {
setPolling(false);
setError(statusData.error || 'Batch reorganization failed');
} else {
// Schedule next poll after this one completes
timeoutId = setTimeout(pollStatus, 500);
}
} catch (err) {
console.error('Failed to get batch reorganization status', err);
if (!cancelled) {
// Retry after delay even on error
timeoutId = setTimeout(pollStatus, 1000);
}
}
};
// Start polling immediately
pollStatus();
return () => {
cancelled = true;
if (timeoutId) {
clearTimeout(timeoutId);
}
};
}, [batchJobId, polling, onComplete]);
const handleClose = useCallback(() => {
// Allow closing - user can reopen to resume if needed
setPreview(null);
setError(null);
setBatchJobId(null);
setStatus(null);
setPolling(false);
onClose();
}, [onClose]);
const handleReorganize = useCallback(async () => {
try {
setError(null);
const result = await executeBatchReorganization(kind);
if (result.batch_job_id) {
setBatchJobId(result.batch_job_id);
setPolling(true);
setStatus({
status: 'pending',
total_collections: result.collection_count,
completed_collections: 0,
current_collection: null,
overall_percent: 0,
completed: [],
failed_collection: null,
error: null,
});
} else {
// No collections needed reorganization
if (onComplete) {
onComplete();
}
handleClose();
}
} catch (err) {
setError(err.message);
}
}, [kind, onComplete, handleClose]);
const renderPreview = () => {
if (!preview) return null;
return (
<Grid columns={1}>
<Grid.Row>
<Grid.Column>
<p>
<strong>{preview.total_collections}</strong> {kindLabelPlural.toLowerCase()} need reorganization,
with <strong>{preview.total_files_needing_move}</strong> total files to move.
</p>
<p>
<strong>New format:</strong> <Label>{preview.new_file_format}</Label>
</p>
</Grid.Column>
</Grid.Row>
{preview.collections && preview.collections.length > 0 && (
<Grid.Row>
<Grid.Column>
<strong>{kindLabelPlural} to reorganize:</strong>
<Table basic='very' compact size='small'>
<Table.Header>
<Table.Row>
<Table.HeaderCell>{kindLabel}</Table.HeaderCell>
<Table.HeaderCell>Files to Move</Table.HeaderCell>
<Table.HeaderCell>Sample</Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>
{preview.collections.slice(0, 10).map((collection, idx) => (
<Table.Row key={idx}>
<Table.Cell>{collection.collection_name}</Table.Cell>
<Table.Cell>
{collection.files_needing_move} / {collection.total_files}
</Table.Cell>
<Table.Cell style={{wordBreak: 'break-all', fontSize: '0.9em'}}>
{collection.sample_move ? (
<>
<span style={{color: '#888'}}>{collection.sample_move.old_path}</span>
<br/>
<Icon name='arrow right' size='small'/>
<span>{collection.sample_move.new_path}</span>
</>
) : '-'}
</Table.Cell>
</Table.Row>
))}
</Table.Body>
</Table>
{preview.collections.length > 10 && (
<p style={{fontStyle: 'italic'}}>
...and {preview.collections.length - 10} more {kindLabelPlural.toLowerCase()}
</p>
)}
</Grid.Column>
</Grid.Row>
)}
{preview.total_collections === 0 && (
<Grid.Row>
<Grid.Column>
<Message info>
<Message.Header>No {kindLabelPlural} Need Reorganization</Message.Header>
<p>All {kindLabelPlural.toLowerCase()} are already organized according to the current format.</p>
</Message>
</Grid.Column>
</Grid.Row>
)}
</Grid>
);
};
const renderProgress = () => {
if (!status) return null;
const currentCollection = status.current_collection;
return (
<Grid columns={1}>
<Grid.Row>
<Grid.Column>
<p>
<strong>Overall Status:</strong> {status.status}
</p>
<strong>Overall Progress:</strong>
<Progress
percent={status.overall_percent || 0}
progress
indicating={status.status !== 'complete' && status.status !== 'failed'}
success={status.status === 'complete'}
error={status.status === 'failed'}
/>
<p>
{status.completed_collections || 0} of {status.total_collections || 0} {kindLabelPlural.toLowerCase()} completed
</p>
</Grid.Column>
</Grid.Row>
{currentCollection && (
<Grid.Row>
<Grid.Column>
<strong>Currently Processing:</strong> {currentCollection.name}
<Progress
percent={currentCollection.percent || 0}
progress
indicating
size='small'
color='blue'
/>
<p style={{fontSize: '0.9em'}}>
{currentCollection.completed || 0} of {currentCollection.total || 0} files
</p>
</Grid.Column>
</Grid.Row>
)}
{status.completed && status.completed.length > 0 && (
<Grid.Row>
<Grid.Column>
<strong>Completed:</strong>
<div style={{maxHeight: '150px', overflowY: 'auto'}}>
{status.completed.map((c, idx) => (
<Label key={idx} color='green' style={{margin: '2px'}}>
<Icon name='check'/> {c.name}
</Label>
))}
</div>
</Grid.Column>
</Grid.Row>
)}
{status.status === 'complete' && (
<Grid.Row>
<Grid.Column>
<Message success>
<Message.Header>Batch Reorganization Complete</Message.Header>
<p>All {kindLabelPlural.toLowerCase()} have been reorganized successfully.</p>
</Message>
</Grid.Column>
</Grid.Row>
)}
{status.status === 'failed' && status.failed_collection && (
<Grid.Row>
<Grid.Column>
<Message negative>
<Message.Header>Reorganization Failed</Message.Header>
<p>
Failed on {kindLabel.toLowerCase()}: <strong>{status.failed_collection.name}</strong>
</p>
{status.error && <p>Error: {status.error}</p>}
</Message>
</Grid.Column>
</Grid.Row>
)}
</Grid>
);
};
const isInProgress = polling || (status && status.status !== 'complete' && status.status !== 'failed');
const canReorganize = preview && preview.total_collections > 0 && !isInProgress && !batchJobId;
return (
<Modal
open={open}
onClose={handleClose}
closeIcon={true}
closeOnDimmerClick={!isInProgress}
closeOnEscape={!isInProgress}
size='large'
>
<Modal.Header>
Reorganize All {kindLabelPlural}
</Modal.Header>
<Modal.Content scrolling>
{loading && (
<Message icon>
<Icon name='circle notched' loading/>
<Message.Content>
<Message.Header>Loading Preview</Message.Header>
Analyzing {kindLabelPlural.toLowerCase()}...
</Message.Content>
</Message>
)}
{error && !status?.failed_collection && (
<Message negative>
<Message.Header>Error</Message.Header>
<p>{error}</p>
</Message>
)}
{!loading && !batchJobId && renderPreview()}
{batchJobId && renderProgress()}
</Modal.Content>
<Modal.Actions>
<Button onClick={handleClose}>
{status?.status === 'complete' ? 'Close' : isInProgress ? 'Hide' : 'Cancel'}
</Button>
{canReorganize && (
<APIButton
color='violet'
onClick={handleReorganize}
obeyWROLMode={true}
>
Reorganize All {kindLabelPlural}
</APIButton>
)}
</Modal.Actions>
</Modal>
);
}

View file

@ -0,0 +1,316 @@
import React from 'react';
import {screen, waitFor, fireEvent} from '@testing-library/react';
import {renderWithProviders} from '../../test-utils';
import {BatchReorganizeModal} from './BatchReorganizeModal';
import * as api from '../../api';
// Mock the API functions
jest.mock('../../api', () => ({
previewBatchReorganization: jest.fn(),
executeBatchReorganization: jest.fn(),
getBatchReorganizationStatus: jest.fn(),
}));
// Mock react-semantic-toasts-2
jest.mock('react-semantic-toasts-2', () => ({
toast: jest.fn(),
}));
// Mock the FileWorkerStatusContext
jest.mock('../../contexts/FileWorkerStatusContext', () => ({
useReorganizationStatus: () => ({
isReorganizing: false,
taskType: null,
collectionId: null,
collectionKind: null,
batchStatus: null,
workerStatus: null,
refresh: jest.fn(),
}),
useFileWorkerStatus: () => ({
status: null,
error: null,
refresh: jest.fn(),
setFastPolling: jest.fn(),
}),
}));
describe('BatchReorganizeModal', () => {
const defaultProps = {
open: true,
onClose: jest.fn(),
kind: 'channel',
onComplete: jest.fn(),
};
beforeEach(() => {
jest.clearAllMocks();
});
it('loads preview when opened', async () => {
const mockPreview = {
collections: [
{
collection_id: 1,
collection_name: 'Test Channel',
total_files: 50,
files_needing_move: 45,
sample_move: {old_path: 'old/path.mp4', new_path: 'new/path.mp4'},
},
],
total_collections: 1,
total_files_needing_move: 45,
new_file_format: '%(upload_date)s/%(title)s.%(ext)s',
};
api.previewBatchReorganization.mockResolvedValue(mockPreview);
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
// Should show loading initially
expect(screen.getByText(/Loading Preview/i)).toBeInTheDocument();
// Wait for preview to load
await waitFor(() => {
expect(screen.getByText(/Test Channel/)).toBeInTheDocument();
});
expect(api.previewBatchReorganization).toHaveBeenCalledWith('channel');
});
it('shows collection list with sample moves', async () => {
const mockPreview = {
collections: [
{
collection_id: 1,
collection_name: 'Channel A',
total_files: 100,
files_needing_move: 80,
sample_move: {old_path: 'videos/old.mp4', new_path: 'videos/new.mp4'},
},
{
collection_id: 2,
collection_name: 'Channel B',
total_files: 50,
files_needing_move: 30,
sample_move: {old_path: 'videos/old2.mp4', new_path: 'videos/new2.mp4'},
},
],
total_collections: 2,
total_files_needing_move: 110,
new_file_format: '%(upload_date)s/%(title)s.%(ext)s',
};
api.previewBatchReorganization.mockResolvedValue(mockPreview);
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
await waitFor(() => {
expect(screen.getByText('Channel A')).toBeInTheDocument();
expect(screen.getByText('Channel B')).toBeInTheDocument();
});
// Should show total statistics - look for the numbers in the text context
expect(screen.getByText('2')).toBeInTheDocument(); // total collections
expect(screen.getByText('110')).toBeInTheDocument(); // total files
});
it('shows total statistics', async () => {
const mockPreview = {
collections: [
{collection_id: 1, collection_name: 'ChannelA', total_files: 50, files_needing_move: 45, sample_move: null},
{collection_id: 2, collection_name: 'ChannelB', total_files: 30, files_needing_move: 25, sample_move: null},
{collection_id: 3, collection_name: 'ChannelC', total_files: 20, files_needing_move: 15, sample_move: null},
],
total_collections: 3,
total_files_needing_move: 85,
new_file_format: '%(title)s.%(ext)s',
};
api.previewBatchReorganization.mockResolvedValue(mockPreview);
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
await waitFor(() => {
// Look for channel names to verify list is displayed
expect(screen.getByText('ChannelA')).toBeInTheDocument();
expect(screen.getByText('ChannelB')).toBeInTheDocument();
expect(screen.getByText('ChannelC')).toBeInTheDocument();
});
});
it('executes batch reorganization on button click', async () => {
const mockPreview = {
collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}],
total_collections: 1,
total_files_needing_move: 5,
new_file_format: '%(title)s.%(ext)s',
};
api.previewBatchReorganization.mockResolvedValue(mockPreview);
api.executeBatchReorganization.mockResolvedValue({
batch_job_id: 'batch-reorganize-abc123',
message: 'Batch reorganization started',
collection_count: 1,
});
api.getBatchReorganizationStatus.mockResolvedValue({
status: 'complete',
total_collections: 1,
completed_collections: 1,
current_collection: null,
overall_percent: 100,
completed: [{id: 1, name: 'Test', status: 'complete'}],
failed_collection: null,
error: null,
});
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
// Wait for preview to load
await waitFor(() => {
expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument();
});
// Click the reorganize button
fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i}));
await waitFor(() => {
expect(api.executeBatchReorganization).toHaveBeenCalledWith('channel');
});
});
it('shows overall progress bar during execution', async () => {
const mockPreview = {
collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}],
total_collections: 1,
total_files_needing_move: 5,
new_file_format: '%(title)s.%(ext)s',
};
api.previewBatchReorganization.mockResolvedValue(mockPreview);
api.executeBatchReorganization.mockResolvedValue({
batch_job_id: 'batch-reorganize-abc123',
message: 'Batch reorganization started',
collection_count: 1,
});
api.getBatchReorganizationStatus.mockResolvedValue({
status: 'running',
total_collections: 1,
completed_collections: 0,
current_collection: {id: 1, name: 'Test', status: 'running', total: 5, completed: 2, percent: 40},
overall_percent: 40,
completed: [],
failed_collection: null,
error: null,
});
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
await waitFor(() => {
expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument();
});
fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i}));
await waitFor(() => {
expect(screen.getByText(/Overall Progress/i)).toBeInTheDocument();
});
});
it('shows per-collection progress bars', async () => {
const mockPreview = {
collections: [{collection_id: 1, collection_name: 'Test Channel', total_files: 10, files_needing_move: 5, sample_move: null}],
total_collections: 1,
total_files_needing_move: 5,
new_file_format: '%(title)s.%(ext)s',
};
api.previewBatchReorganization.mockResolvedValue(mockPreview);
api.executeBatchReorganization.mockResolvedValue({
batch_job_id: 'batch-reorganize-abc123',
message: 'Batch reorganization started',
collection_count: 1,
});
api.getBatchReorganizationStatus.mockResolvedValue({
status: 'running',
total_collections: 1,
completed_collections: 0,
current_collection: {id: 1, name: 'Test Channel', status: 'running', total: 5, completed: 2, percent: 40},
overall_percent: 20,
completed: [],
failed_collection: null,
error: null,
});
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
await waitFor(() => {
expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument();
});
fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i}));
await waitFor(() => {
expect(screen.getByText(/Currently Processing/i)).toBeInTheDocument();
expect(screen.getByText(/Test Channel/)).toBeInTheDocument();
});
});
it('calls onComplete when finished', async () => {
const mockOnComplete = jest.fn();
const mockPreview = {
collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}],
total_collections: 1,
total_files_needing_move: 5,
new_file_format: '%(title)s.%(ext)s',
};
api.previewBatchReorganization.mockResolvedValue(mockPreview);
api.executeBatchReorganization.mockResolvedValue({
batch_job_id: 'batch-reorganize-abc123',
message: 'Batch reorganization started',
collection_count: 1,
});
api.getBatchReorganizationStatus.mockResolvedValue({
status: 'complete',
total_collections: 1,
completed_collections: 1,
current_collection: null,
overall_percent: 100,
completed: [{id: 1, name: 'Test', status: 'complete'}],
failed_collection: null,
error: null,
});
renderWithProviders(<BatchReorganizeModal {...defaultProps} onComplete={mockOnComplete} />);
await waitFor(() => {
expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument();
});
fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i}));
await waitFor(() => {
expect(mockOnComplete).toHaveBeenCalled();
});
});
it('handles domain kind correctly', async () => {
const mockPreview = {
collections: [{collection_id: 1, collection_name: 'example.com', total_files: 10, files_needing_move: 5, sample_move: null}],
total_collections: 1,
total_files_needing_move: 5,
new_file_format: '%(title)s.%(ext)s',
};
api.previewBatchReorganization.mockResolvedValue(mockPreview);
renderWithProviders(<BatchReorganizeModal {...defaultProps} kind="domain" />);
await waitFor(() => {
expect(screen.getByText(/Reorganize All Domains/i)).toBeInTheDocument();
});
expect(api.previewBatchReorganization).toHaveBeenCalledWith('domain');
});
});

View file

@ -0,0 +1,345 @@
import React, {useCallback, useEffect, useState} from 'react';
import {Grid, Icon, Label, Progress} from 'semantic-ui-react';
import Message from 'semantic-ui-react/dist/commonjs/collections/Message';
import {Button, Modal, Table} from '../Theme';
import {APIButton} from '../Common';
import {
executeCollectionReorganization,
getReorganizationStatus,
previewCollectionReorganization
} from '../../api';
import {useReorganizationStatus} from '../../contexts/FileWorkerStatusContext';
/**
* Modal component for previewing and executing collection file reorganization.
*
* Shows a preview of files that will be moved when the file_name_format changes,
* allows user to execute the reorganization, and tracks progress.
*
* @param {boolean} open - Whether the modal is open
* @param {Function} onClose - Callback when modal is closed
* @param {number} collectionId - ID of the collection to reorganize
* @param {string} collectionName - Name of the collection for display
* @param {Function} onComplete - Optional callback when reorganization completes
* @param {boolean} needsReorganization - Whether the collection.needs_reorganization flag is true
*/
export function CollectionReorganizeModal({
open,
onClose,
collectionId,
collectionName,
onComplete,
needsReorganization,
}) {
const [loading, setLoading] = useState(false);
const [preview, setPreview] = useState(null);
const [error, setError] = useState(null);
const [jobId, setJobId] = useState(null);
const [status, setStatus] = useState(null);
const [polling, setPolling] = useState(false);
// Check if this collection is currently being reorganized
const {
isReorganizing,
taskType,
collectionId: activeCollectionId,
workerStatus,
} = useReorganizationStatus();
// Load preview when modal opens, or resume if already reorganizing
useEffect(() => {
if (open && collectionId) {
// Check if THIS collection is currently being reorganized
const isThisActive = isReorganizing &&
taskType === 'reorganize' &&
activeCollectionId === collectionId;
if (isThisActive) {
// Resume: show current progress from worker status
setLoading(false);
setPreview(null);
setJobId('active'); // Marker for active job
setStatus({
status: workerStatus.status,
total: workerStatus.operation_total,
completed: workerStatus.operation_processed,
percent: workerStatus.operation_percent,
});
setPolling(true);
} else if (!polling && !jobId) {
// Normal: fetch preview (only if not already polling/resuming)
setLoading(true);
setError(null);
setPreview(null);
setJobId(null);
setStatus(null);
previewCollectionReorganization(collectionId)
.then(data => {
setPreview(data);
setLoading(false);
})
.catch(err => {
setError(err.message);
setLoading(false);
});
}
}
}, [open, collectionId, isReorganizing, taskType, activeCollectionId]);
// Poll for status when we have a job ID
// Uses setTimeout instead of setInterval to wait for each request to complete
// before scheduling the next one, preventing request pile-up on slow connections
useEffect(() => {
if (!jobId || !polling) return;
let timeoutId = null;
let cancelled = false;
const pollStatus = async () => {
if (cancelled) return;
try {
const statusData = await getReorganizationStatus(collectionId, jobId);
if (cancelled) return;
setStatus(statusData);
if (statusData.status === 'complete') {
setPolling(false);
if (onComplete) {
onComplete();
}
} else if (statusData.status === 'error' || statusData.error) {
setPolling(false);
setError(statusData.error || 'Reorganization failed');
} else {
// Schedule next poll after this one completes
timeoutId = setTimeout(pollStatus, 1000);
}
} catch (err) {
console.error('Failed to get reorganization status', err);
if (!cancelled) {
// Retry after delay even on error
timeoutId = setTimeout(pollStatus, 1000);
}
}
};
// Start polling immediately
pollStatus();
return () => {
cancelled = true;
if (timeoutId) {
clearTimeout(timeoutId);
}
};
}, [jobId, polling, collectionId, onComplete]);
const handleClose = useCallback(() => {
// Allow closing - user can reopen to resume if needed
setPreview(null);
setError(null);
setJobId(null);
setStatus(null);
setPolling(false);
onClose();
}, [onClose]);
const handleReorganize = useCallback(async () => {
try {
setError(null);
const result = await executeCollectionReorganization(collectionId);
if (result.job_id) {
setJobId(result.job_id);
setPolling(true);
setStatus({status: 'pending', total: 0, completed: 0, percent: 0});
} else {
// No files needed reorganization
if (onComplete) {
onComplete();
}
handleClose();
}
} catch (err) {
setError(err.message);
}
}, [collectionId, onComplete, handleClose]);
const renderPreview = () => {
if (!preview) return null;
return (
<Grid columns={1}>
{/* Warning when collection appears organized but has files to move */}
{preview.files_needing_move > 0 && !needsReorganization && (
<Grid.Row>
<Grid.Column>
<Message warning>
<Message.Header>Collection Appears Organized</Message.Header>
<p>
This collection's format matches the current configuration, but
{' '}<strong>{preview.files_needing_move}</strong> files don't match the expected layout.
This can happen if a previous reorganization was interrupted.
</p>
</Message>
</Grid.Column>
</Grid.Row>
)}
<Grid.Row>
<Grid.Column>
<p>
<strong>{preview.files_needing_move}</strong> of <strong>{preview.total_files}</strong> files
will be reorganized.
</p>
{preview.current_file_format && (
<p>
<strong>Current format:</strong> <code>{preview.current_file_format}</code>
</p>
)}
<p>
<strong>New format:</strong> <Label>{preview.new_file_format}</Label>
</p>
</Grid.Column>
</Grid.Row>
{preview.sample_moves && preview.sample_moves.length > 0 && (
<Grid.Row>
<Grid.Column>
<strong>Sample moves:</strong>
<Table basic='very' compact size='small'>
<Table.Header>
<Table.Row>
<Table.HeaderCell>Current Path</Table.HeaderCell>
<Table.HeaderCell/>
<Table.HeaderCell>New Path</Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>
{preview.sample_moves.map((move, idx) => (
<Table.Row key={idx}>
<Table.Cell style={{wordBreak: 'break-all'}}>
{move.old_path}
</Table.Cell>
<Table.Cell>
<Icon name='arrow right'/>
</Table.Cell>
<Table.Cell style={{wordBreak: 'break-all'}}>
{move.new_path}
</Table.Cell>
</Table.Row>
))}
</Table.Body>
</Table>
{preview.files_needing_move > preview.sample_moves.length && (
<p style={{fontStyle: 'italic'}}>
...and {preview.files_needing_move - preview.sample_moves.length} more files
</p>
)}
</Grid.Column>
</Grid.Row>
)}
{preview.files_needing_move === 0 && (
<Grid.Row>
<Grid.Column>
<Message info>
<Message.Header>No Files Need Reorganization</Message.Header>
<p>All files are already organized according to the current format.</p>
</Message>
</Grid.Column>
</Grid.Row>
)}
</Grid>
);
};
const renderProgress = () => {
if (!status) return null;
return (
<Grid columns={1}>
<Grid.Row>
<Grid.Column>
<p>
<strong>Status:</strong> {status.status}
</p>
<Progress
percent={status.percent || 0}
progress
indicating={status.status !== 'complete'}
success={status.status === 'complete'}
/>
<p>
{status.completed || 0} of {status.total || 0} files processed
</p>
</Grid.Column>
</Grid.Row>
{status.status === 'complete' && (
<Grid.Row>
<Grid.Column>
<Message success>
<Message.Header>Reorganization Complete</Message.Header>
<p>All files have been reorganized successfully.</p>
</Message>
</Grid.Column>
</Grid.Row>
)}
</Grid>
);
};
const isInProgress = polling || (status && status.status !== 'complete' && status.status !== 'error');
const canReorganize = preview && preview.files_needing_move > 0 && !isInProgress && !jobId;
return (
<Modal
open={open}
onClose={handleClose}
closeIcon={true}
closeOnDimmerClick={!isInProgress}
closeOnEscape={!isInProgress}
>
<Modal.Header>
Reorganize Files: {collectionName}
</Modal.Header>
<Modal.Content>
{loading && (
<Message icon>
<Icon name='circle notched' loading/>
<Message.Content>
<Message.Header>Loading Preview</Message.Header>
Analyzing files...
</Message.Content>
</Message>
)}
{error && (
<Message negative>
<Message.Header>Error</Message.Header>
<p>{error}</p>
</Message>
)}
{!loading && !jobId && renderPreview()}
{jobId && renderProgress()}
</Modal.Content>
<Modal.Actions>
<Button onClick={handleClose}>
{status?.status === 'complete' ? 'Close' : isInProgress ? 'Hide' : 'Cancel'}
</Button>
{canReorganize && (
<APIButton
color='violet'
onClick={handleReorganize}
obeyWROLMode={true}
>
Reorganize Files
</APIButton>
)}
</Modal.Actions>
</Modal>
);
}

View file

@ -0,0 +1,67 @@
import React, {createContext, useContext, useState, useEffect, useCallback, useRef} from 'react';
import {fetchFilesProgress} from '../api';
const FileWorkerStatusContext = createContext(null);
export function FileWorkerStatusProvider({children, pollInterval = 3000}) {
const [status, setStatus] = useState(null);
const [error, setError] = useState(null);
const requestedInterval = useRef(pollInterval);
const refresh = useCallback(async () => {
try {
const data = await fetchFilesProgress();
setStatus(data);
setError(null);
} catch (err) {
setError(err);
}
}, []);
// Allow components to request faster polling temporarily
const setFastPolling = useCallback((fast) => {
requestedInterval.current = fast ? 500 : pollInterval;
}, [pollInterval]);
useEffect(() => {
let timeoutId;
const poll = async () => {
await refresh();
timeoutId = setTimeout(poll, requestedInterval.current);
};
poll();
return () => clearTimeout(timeoutId);
}, [refresh]);
return (
<FileWorkerStatusContext.Provider value={{status, error, refresh, setFastPolling}}>
{children}
</FileWorkerStatusContext.Provider>
);
}
export function useFileWorkerStatus() {
const context = useContext(FileWorkerStatusContext);
if (!context) {
throw new Error('useFileWorkerStatus must be used within FileWorkerStatusProvider');
}
return context;
}
// Convenience hook for reorganization-specific status
export function useReorganizationStatus() {
const {status: progress, refresh} = useFileWorkerStatus();
const isReorganizing = progress?.status === 'reorganizing' ||
progress?.status === 'batch_reorganizing';
return {
isReorganizing,
taskType: progress?.task_type,
collectionId: progress?.collection_id,
collectionKind: progress?.collection_kind,
batchStatus: progress?.batch_status,
workerStatus: progress,
refresh,
};
}

View file

@ -5,7 +5,6 @@ import {
fetchChannels,
fetchDecoded,
fetchDomains,
fetchFilesProgress,
filesSearch,
getArchive,
getChannel,
@ -588,7 +587,7 @@ export const useChannel = (channel_id) => {
return {
channel: form.formData,
form,
fetchChannel,
fetchChannel: form.fetcher,
};
}
@ -760,34 +759,6 @@ export const useBrowseFiles = () => {
return {browseFiles, openFolders, setOpenFolders, fetchFiles};
}
export const useFilesProgress = () => {
const [progress, setProgress] = useState(null);
const localFetchFilesProgress = async () => {
try {
let p = await fetchFilesProgress();
setProgress(p);
} catch (e) {
setProgress(null);
console.error(e);
}
}
useEffect(() => {
localFetchFilesProgress();
}, []);
return {progress, fetchFilesProgress: localFetchFilesProgress}
}
export const useFilesProgressInterval = () => {
const {progress, fetchFilesProgress} = useFilesProgress();
useRecurringTimeout(fetchFilesProgress, 1000);
return {progress, fetchFilesProgress};
}
export const useHotspot = () => {
const [on, setOn] = useState(null);
const [inUse, setInUse] = useState(false);

View file

@ -3,11 +3,50 @@
* Provides custom render functions with necessary context providers
*/
import React from 'react';
import React, {createContext, useContext} from 'react';
import {render} from '@testing-library/react';
import {BrowserRouter} from 'react-router';
import {MediaContextProvider, ThemeContext} from './contexts/contexts';
// Mock FileWorkerStatusContext for tests - avoids API calls during testing
const MockFileWorkerStatusContext = createContext(null);
function MockFileWorkerStatusProvider({children}) {
const mockValue = {
status: null,
error: null,
refresh: jest.fn(),
setFastPolling: jest.fn(),
};
return (
<MockFileWorkerStatusContext.Provider value={mockValue}>
{children}
</MockFileWorkerStatusContext.Provider>
);
}
// Re-export hooks that use the mock context
export function useFileWorkerStatus() {
const context = useContext(MockFileWorkerStatusContext);
if (!context) {
throw new Error('useFileWorkerStatus must be used within FileWorkerStatusProvider');
}
return context;
}
export function useReorganizationStatus() {
const {status: progress, refresh} = useFileWorkerStatus();
return {
isReorganizing: false,
taskType: null,
collectionId: null,
collectionKind: null,
batchStatus: null,
workerStatus: progress,
refresh,
};
}
/**
* Custom render function that wraps components with necessary providers
* @param {React.Component} ui - Component to render
@ -40,9 +79,11 @@ export function renderWithProviders(
function Wrapper({children}) {
const content = (
<BrowserRouter>
<ThemeContext.Provider value={defaultThemeContext}>
{children}
</ThemeContext.Provider>
<MockFileWorkerStatusProvider>
<ThemeContext.Provider value={defaultThemeContext}>
{children}
</ThemeContext.Provider>
</MockFileWorkerStatusProvider>
</BrowserRouter>
);

View file

@ -7,7 +7,7 @@ from typing import List, Tuple, Iterable
from sqlalchemy import not_
from sqlalchemy.orm import Session
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM, FIREFOX
from wrolpi.collections import Collection
from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
slow_logger, get_title_from_html, TRACE_LEVEL
@ -86,11 +86,29 @@ class ArchiveDownloader(Downloader, ABC):
async def do_singlefile(self, download: Download) -> bytes:
"""Create a Singlefile from the archive's URL."""
# Get browser settings from config
config = lib.get_archive_downloader_config()
# Determine browser executable
browser_executable = config.browser_executable
if browser_executable:
# Use configured browser path
browser = browser_executable
else:
# Auto-detect: prefer Chromium, fallback to Firefox
browser = CHROMIUM or FIREFOX
if not browser:
raise RuntimeError('No browser found. Install Chromium or Firefox, or configure browser_executable.')
# Get browser args and user agent from config
browser_args = config.browser_args or '["--no-sandbox"]'
user_agent = config.user_agent or DOWNLOAD_USER_AGENT
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
str(SINGLE_FILE_BIN),
'--browser-executable-path', CHROMIUM,
'--browser-args', '["--no-sandbox"]',
'--user-agent', DOWNLOAD_USER_AGENT,
'--browser-executable-path', str(browser),
'--browser-args', browser_args,
'--user-agent', user_agent,
'--dump-content',
'--load-deferred-images-dispatch-scroll-event',
download.url,
@ -233,17 +251,7 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive:
file_group.title = file_group.a_text = title or archive.file_group.title
file_group.d_text = contents
# Store paths as filenames only (relative to directory), resolved via file_group.resolve_path()
file_group.data = {
'id': archive.id,
'domain': archive.domain,
'readability_json_path': archive.readability_json_path.name if archive.readability_json_path else None,
'readability_path': archive.readability_path.name if archive.readability_path else None,
'readability_txt_path': archive.readability_txt_path.name if archive.readability_txt_path else None,
'screenshot_path': archive.screenshot_path.name if archive.screenshot_path else None,
'singlefile_path': archive.singlefile_path.name if archive.singlefile_path else None,
'info_json_path': archive.info_json_path.name if archive.info_json_path else None,
}
archive.rebuild_data()
return archive
except Exception as e:

View file

@ -7,12 +7,14 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response, api_app
from wrolpi.cmd import get_installed_browsers
from wrolpi.common import logger, wrol_mode_check, api_param_limiter, TRACE_LEVEL
from wrolpi.db import get_db_session
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import DOCKERIZED
from . import lib, schema
NAME = 'archive'
@ -90,6 +92,28 @@ async def post_file_format(_: Request, body: schema.ArchiveFileFormatRequest):
), status=HTTPStatus.BAD_REQUEST)
@archive_bp.get('/browsers')
@openapi.definition(
summary='Get list of installed browsers that can be used for archiving.',
description='Returns browsers installed on the system. Only available on native deployments (not Docker).',
)
async def get_browsers(_: Request):
"""Get list of installed browsers for SingleFile."""
if DOCKERIZED:
# Browser configuration is not available in Docker
return json_response(dict(
browsers=[],
available=False,
message='Browser configuration is not available in Docker deployments.',
))
browsers = get_installed_browsers()
return json_response(dict(
browsers=browsers,
available=True,
))
@archive_bp.get('/upload')
@openapi.definition(
summary='A message to confirm to the user that they have the correct upload URL.'

View file

@ -22,7 +22,7 @@ from wrolpi.cmd import READABILITY_BIN, run_command
from wrolpi.collections import Collection
from wrolpi.common import get_media_directory, get_relative_to_media_directory, logger, extract_domain, \
escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \
get_wrolpi_config, html_screenshot, ConfigFile
get_wrolpi_config, html_screenshot, ConfigFile, trim_file_name
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
@ -218,12 +218,26 @@ class ArchiveDownloaderConfigValidator:
"""Validator for archives_downloader.yaml config."""
file_name_format: str
version: int = 0
# Browser settings (only used on native deployments, not Docker)
browser_executable: str = None # None means auto-detect, or absolute path
browser_args: str = '["--no-sandbox"]' # JSON array of browser arguments
user_agent: str = None # None means use system default
def __post_init__(self):
# Validate file_name_format ends with .%(ext)s (like video format)
if not self.file_name_format.endswith('.%(ext)s'):
raise ValueError('file_name_format must end with .%(ext)s')
# Validate browser_args is valid JSON array
if self.browser_args:
import json
try:
args = json.loads(self.browser_args)
if not isinstance(args, list):
raise ValueError('browser_args must be a JSON array')
except json.JSONDecodeError as e:
raise ValueError(f'browser_args must be valid JSON: {e}')
class ArchiveDownloaderConfig(ConfigFile):
"""
@ -235,6 +249,9 @@ class ArchiveDownloaderConfig(ConfigFile):
Format:
version: 0
file_name_format: '%(download_date)s_%(title)s.%(ext)s'
browser_executable: null # null for auto-detect, or absolute path
browser_args: '["--no-sandbox"]' # JSON array of browser arguments
user_agent: null # null for system default
Variables available in file_name_format:
- %(title)s - Page title (extracted from HTML)
@ -248,18 +265,41 @@ class ArchiveDownloaderConfig(ConfigFile):
Subdirectories can be included in the format, e.g.:
'%(download_year)s/%(download_datetime)s_%(title)s.%(ext)s'
Browser settings (only used on native deployments, not Docker):
- browser_executable: Path to browser or null for auto-detect
- browser_args: JSON array of arguments passed to browser
- user_agent: Custom user agent string or null for default
"""
file_name = 'archives_downloader.yaml'
validator = ArchiveDownloaderConfigValidator
default_config = dict(
version=0,
file_name_format='%(download_datetime)s_%(title)s.%(ext)s',
browser_executable=None,
browser_args='["--no-sandbox"]',
user_agent=None,
)
@property
def file_name_format(self) -> str:
return self._config['file_name_format']
@property
def browser_executable(self) -> Optional[str]:
"""Get the configured browser executable path, or None for auto-detect."""
return self._config.get('browser_executable')
@property
def browser_args(self) -> str:
"""Get the browser arguments as a JSON string."""
return self._config.get('browser_args', '["--no-sandbox"]')
@property
def user_agent(self) -> Optional[str]:
"""Get the custom user agent, or None for system default."""
return self._config.get('user_agent')
def import_config(self, file: pathlib.Path = None, send_events=False):
super().import_config(file, send_events)
self.successful_import = True
@ -314,11 +354,20 @@ def format_archive_filename(
)
try:
return template % variables
result = template % variables
# Trim the filename portion if it's too long (may include subdirectory paths)
if '/' in result:
# Has subdirectories - only trim the filename part
parts = result.rsplit('/', 1)
subdir = parts[0]
filename = trim_file_name(parts[1])
return f'{subdir}/{filename}'
else:
return trim_file_name(result)
except KeyError as e:
logger.error(f'Invalid variable in archive file_name_format: {e}')
# Fallback to default format
return f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html'
return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
def preview_archive_filename(file_name_format: str) -> str:
@ -356,6 +405,69 @@ def preview_archive_filename(file_name_format: str) -> str:
raise RuntimeError(f'Invalid format: {e}')
def format_archive_filename_from_archive(
archive: Archive,
file_name_format: str = None,
) -> str:
"""Single source of truth for archive filenames when reorganizing.
Formats an archive filename using the configured file_name_format template.
Used by collection reorganization to compute new file paths.
Args:
archive: Archive model with metadata
file_name_format: Optional override, defaults to config value
Returns:
Formatted filename (may include subdirectory path like "2025/My Article.html")
"""
config = get_archive_downloader_config()
template = file_name_format or config.file_name_format
file_group = archive.file_group
# Get download date from file_group
download_date = file_group.download_datetime or file_group.published_datetime or now()
# Get title
title = file_group.title or 'untitled'
# Get domain from URL
domain = ''
if file_group.url:
try:
domain = extract_domain(file_group.url)
except Exception:
pass
variables = dict(
title=escape_file_name(title) if title else 'untitled',
domain=domain,
download_datetime=archive_strftime(download_date),
download_date=download_date.strftime('%Y-%m-%d'),
download_year=str(download_date.year),
download_month=f'{download_date.month:02d}',
download_day=f'{download_date.day:02d}',
ext='html',
)
try:
result = template % variables
# Trim the filename portion if it's too long (may include subdirectory paths)
if '/' in result:
# Has subdirectories - only trim the filename part
parts = result.rsplit('/', 1)
subdir = parts[0]
filename = trim_file_name(parts[1])
return f'{subdir}/{filename}'
else:
return trim_file_name(result)
except KeyError as e:
logger.error(f'Invalid variable in archive file_name_format: {e}')
# Fallback to default format
return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
@dataclass
class ArchiveFiles:
"""Every Archive will have some of these files."""
@ -416,17 +528,18 @@ def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.P
full_path.parent.mkdir(parents=True, exist_ok=True)
singlefile_path = full_path
stem = full_path.stem
readability_path = full_path.parent / f'{stem}.readability.html'
readability_txt_path = full_path.parent / f'{stem}.readability.txt'
readability_json_path = full_path.parent / f'{stem}.readability.json'
screenshot_path = full_path.parent / f'{stem}.png'
parent_dir = full_path.parent
readability_path = trim_file_name(parent_dir / f'{stem}.readability.html')
readability_txt_path = trim_file_name(parent_dir / f'{stem}.readability.txt')
readability_json_path = trim_file_name(parent_dir / f'{stem}.readability.json')
screenshot_path = trim_file_name(parent_dir / f'{stem}.png')
else:
singlefile_path = directory / formatted_filename
stem = singlefile_path.stem
readability_path = directory / f'{stem}.readability.html'
readability_txt_path = directory / f'{stem}.readability.txt'
readability_json_path = directory / f'{stem}.readability.json'
screenshot_path = directory / f'{stem}.png'
readability_path = trim_file_name(directory / f'{stem}.readability.html')
readability_txt_path = trim_file_name(directory / f'{stem}.readability.txt')
readability_json_path = trim_file_name(directory / f'{stem}.readability.json')
screenshot_path = trim_file_name(directory / f'{stem}.png')
paths = (singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path)
@ -592,6 +705,15 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
archive.file_group.download_datetime = now()
archive.url = url
archive.collection = get_or_create_domain_collection(session, url)
# Update collection's file_format to track the format used for this file
config = get_archive_downloader_config()
if archive.collection and config.file_name_format:
if not archive.collection.file_format:
# Set initial file_format on collection
archive.collection.file_format = config.file_name_format
save_domains_config.activate_switch()
archive.flush()
return archive

View file

@ -351,6 +351,23 @@ class Archive(Base, ModelHelper):
if metadata.modified_datetime:
self.file_group.published_modified_datetime = metadata.modified_datetime
def rebuild_data(self):
"""Rebuild the data field with current file paths.
Called during validate() to ensure data field reflects actual filenames
after reorganization or other file operations.
"""
self.file_group.data = {
'id': self.id,
'domain': self.domain,
'readability_json_path': self.readability_json_path.name if self.readability_json_path else None,
'readability_path': self.readability_path.name if self.readability_path else None,
'readability_txt_path': self.readability_txt_path.name if self.readability_txt_path else None,
'screenshot_path': self.screenshot_path.name if self.screenshot_path else None,
'singlefile_path': self.singlefile_path.name if self.singlefile_path else None,
'info_json_path': self.info_json_path.name if self.info_json_path else None,
}
def validate(self):
"""Fill in any missing data about this Archive from its files."""
try:
@ -359,6 +376,7 @@ class Archive(Base, ModelHelper):
self.apply_domain()
self.apply_singlefile_title()
self.apply_metadata()
self.rebuild_data()
except Exception as e:
logger.warning(f'Unable to validate {self}', exc_info=e)
if PYTEST:

View file

@ -342,18 +342,9 @@ async def test_archive_generate_screenshot(test_session, async_client, archive_f
assert original_screenshot_path is not None, 'Archive should have a screenshot'
assert original_screenshot_path.is_file(), 'Screenshot file should exist'
# Simulate the bug: archive has screenshot file but data['screenshot_path'] is not set
# (This could happen if the archive was created before set_screenshot was implemented)
if archive.file_group.data and 'screenshot_path' in archive.file_group.data:
data = dict(archive.file_group.data)
del data['screenshot_path']
archive.file_group.data = data
test_session.commit()
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Verify data was cleared but file still exists
assert 'screenshot_path' not in (archive.file_group.data or {}), 'screenshot_path should not be in data yet'
assert archive.screenshot_path is not None, 'But screenshot file should still exist'
# Note: Previously this test simulated a bug where data['screenshot_path'] could be missing
# while the file existed. This scenario is now auto-fixed by Archive.validate() which calls
# rebuild_data(), so the simulation is no longer necessary.
request, response = await async_client.post(f'/api/archive/{archive.id}/generate_screenshot')
assert response.status_code == HTTPStatus.OK
@ -364,11 +355,11 @@ async def test_archive_generate_screenshot(test_session, async_client, archive_f
assert archive.screenshot_path == original_screenshot_path, 'Screenshot path should be unchanged'
assert archive.screenshot_path.is_file(), 'Screenshot file should still exist'
assert archive.screenshot_file is not None, 'Screenshot file should still be tracked'
# IMPORTANT: Also verify FileGroup.data was updated (this catches the bug where only files was updated)
# IMPORTANT: Also verify FileGroup.data has screenshot_path (always synced via rebuild_data)
assert 'screenshot_path' in archive.file_group.data, \
'screenshot_path should be in FileGroup.data after ensuring tracking'
assert str(archive.file_group.data['screenshot_path']) == str(original_screenshot_path), \
'screenshot_path in data should match the file path'
'screenshot_path should be in FileGroup.data'
assert archive.file_group.data['screenshot_path'] == original_screenshot_path.name, \
'screenshot_path in data should be filename only'
# Test error case: Archive has no singlefile
archive_no_singlefile = archive_factory('example.com', 'https://example.com/no-singlefile', screenshot=False)
@ -428,3 +419,28 @@ async def test_archive_file_format_preview(async_client):
request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'error' in response.json
@pytest.mark.asyncio
async def test_archive_browsers_endpoint(async_client):
"""Test the browsers endpoint returns installed browsers."""
# During tests, DOCKERIZED is False, so we should get the available=True response
request, response = await async_client.get('/api/archive/browsers')
assert response.status_code == HTTPStatus.OK
assert 'browsers' in response.json
assert 'available' in response.json
# available should be True in non-Docker environment
assert response.json['available'] is True
# browsers is a list
assert isinstance(response.json['browsers'], list)
@pytest.mark.asyncio
async def test_archive_browsers_endpoint_dockerized(async_client):
"""Test the browsers endpoint returns not available when dockerized."""
with patch('modules.archive.api.DOCKERIZED', True):
request, response = await async_client.get('/api/archive/browsers')
assert response.status_code == HTTPStatus.OK
assert response.json['browsers'] == []
assert response.json['available'] is False
assert 'message' in response.json

View file

@ -365,14 +365,17 @@ async def test_get_title_from_html(test_directory, test_session, fake_now, async
@skip_circleci
def test_get_new_archive_files_length(test_directory, fake_now):
"""Archive titles are truncated to fit file system length limit. (255-character limit for most file systems)"""
"""Archive titles are truncated to fit file system BYTE limit (200 bytes - see MAXIMUM_FILE_BYTES)."""
from wrolpi.common import MAXIMUM_FILE_BYTES
fake_now(datetime(2001, 1, 1))
archive_files = get_new_archive_files('https://example.com', 'a' * 500)
assert len(str(archive_files.singlefile.name)) == 225
assert len(str(archive_files.readability.name)) == 237
assert len(str(archive_files.readability_txt.name)) == 236
assert len(str(archive_files.readability_json.name)) == 237
assert len(str(archive_files.screenshot.name)) == 224
# All filenames should be <= MAXIMUM_FILE_BYTES (bytes, not characters)
assert len(str(archive_files.singlefile.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
assert len(str(archive_files.readability.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
assert len(str(archive_files.readability_txt.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
assert len(str(archive_files.readability_json.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
assert len(str(archive_files.screenshot.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
@skip_circleci
@ -1289,6 +1292,26 @@ def test_archive_downloader_config_validation():
with pytest.raises(ValueError, match='must end with'):
ArchiveDownloaderConfigValidator(file_name_format='%(download_date)s_%(title)s')
# Valid browser_args (JSON array)
ArchiveDownloaderConfigValidator(
file_name_format='%(title)s.%(ext)s',
browser_args='["--no-sandbox", "--headless"]'
)
# Invalid browser_args (not valid JSON)
with pytest.raises(ValueError, match='must be valid JSON'):
ArchiveDownloaderConfigValidator(
file_name_format='%(title)s.%(ext)s',
browser_args='not valid json'
)
# Invalid browser_args (not an array)
with pytest.raises(ValueError, match='must be a JSON array'):
ArchiveDownloaderConfigValidator(
file_name_format='%(title)s.%(ext)s',
browser_args='{"key": "value"}'
)
def test_format_archive_filename_default(test_directory, fake_now):
"""format_archive_filename uses default format."""

View file

@ -128,3 +128,53 @@ async def test_set_screenshot(async_client, test_session, archive_factory, image
# Verify no screenshot was set
assert archive_no_screenshot.screenshot_path is None
@pytest.mark.asyncio
async def test_archive_validate_rebuilds_data_after_filename_change(async_client, test_session, test_directory,
archive_factory):
"""Archive.validate() should rebuild data field when filenames change.
This tests the bug where reorganization updates files[] but not data{}.
The data field contains cached path names that must stay in sync.
The bug scenario: files are renamed during reorganization (e.g., fixing separator
from '___' to '_'), files[] is updated but data{} is not. The UI then uses
stale data.singlefile_path and gets 404 errors.
"""
# Create an archive using the factory
archive = archive_factory(domain='test.com', url='https://test.com/article')
test_session.commit()
# Verify initial data field has singlefile_path
assert archive.file_group.data is not None
original_singlefile = archive.file_group.data.get('singlefile_path')
assert original_singlefile is not None
# Simulate what reorganization does: update files[] but not data{}
# (This mimics _bulk_update_file_groups_reorganize behavior)
# Use a realistic rename: change the title part of the filename
# Original: '2000-01-01-07-00-01_NA.html' -> '2000-01-01-07-00-01_New-Title.html'
new_filename = original_singlefile.replace('_NA.html', '_New-Title.html')
old_files = archive.file_group.files
new_files = []
for f in old_files:
new_f = dict(f)
if f['path'].endswith('.html') and not f['path'].endswith('.readability.html'):
new_f['path'] = new_filename
new_files.append(new_f)
archive.file_group.files = new_files
archive.file_group.primary_path = archive.file_group.directory / new_filename
# Rename the actual file on disk to match the new filename
old_path = archive.file_group.directory / original_singlefile
new_path = archive.file_group.directory / new_filename
old_path.rename(new_path)
# data field still has OLD filename (this is the bug)
assert archive.file_group.data['singlefile_path'] == original_singlefile
# After validate(), data field should have NEW filename
archive.validate()
assert archive.file_group.data['singlefile_path'] == new_filename

View file

@ -260,6 +260,9 @@ async def test_channel_post_directory(test_session, async_client, test_directory
async def test_channel_by_id(test_session, async_client, simple_channel, simple_video):
request, response = await async_client.get(f'/api/videos/channels/{simple_channel.id}')
assert response.status_code == HTTPStatus.OK
channel = response.json['channel']
# needs_reorganization should be included for UI to display reorganization status
assert 'needs_reorganization' in channel
@pytest.mark.asyncio

View file

@ -139,7 +139,10 @@ def convert_wrolpi_filename_format(filename_format: str) -> str:
def preview_filename(filename_format: str) -> str:
"""Return an example video file name formatted using the provided filename format.
"""Return an example video file name using simple string formatting.
This is a lightweight implementation that doesn't require YoutubeDL instantiation,
which is expensive (~1500 extractors loaded, regexes compiled).
@raise RuntimeError: If the format is invalid.
"""
@ -149,21 +152,31 @@ def preview_filename(filename_format: str) -> str:
# Convert WROLPi-specific variables to yt-dlp syntax
converted_format = convert_wrolpi_filename_format(filename_format)
options = get_videos_downloader_config().yt_dlp_options
options['outtmpl'] = converted_format
ydl = YoutubeDL(copy.deepcopy(options))
entry = dict(
sample_date = now()
variables = dict(
uploader='WROLPi',
timestamp=int(now().timestamp()),
upload_date=now().strftime('%Y%m%d'),
channel='WROLPi',
upload_date=sample_date.strftime('%Y%m%d'),
id='Qz-FuenRylQ',
title='The title of the video',
ext='mp4',
description='A description of the video',
timestamp=int(sample_date.timestamp()),
)
filename = prepare_filename(entry, ydl=ydl).lstrip('/')
return filename
# Handle yt-dlp's date extraction syntax %(upload_date>%Y)s and %(upload_date>%m)s
# Replace these with their values before standard Python % formatting
def replace_date_format(match):
date_format = match.group(1)
return sample_date.strftime(date_format)
converted_format = re.sub(r'%\(upload_date>([^)]+)\)s', replace_date_format, converted_format)
try:
filename = converted_format % variables
return filename.lstrip('/')
except KeyError as e:
raise RuntimeError(f'Invalid variable: {e}')
@cached_multiprocessing_result
@ -545,6 +558,17 @@ class VideoDownloader(Downloader, ABC):
video_id, video_info_json_path = video.id, video.info_json_path
if video_info_json_path and (new_info_json := video.clean_info_json()):
video.replace_info_json(new_info_json, clean=False)
# Update channel's collection's file_format to track the format used
if channel_id:
from modules.videos.models import Channel
from modules.videos.lib import save_channels_config
channel_obj = session.query(Channel).get(channel_id)
if channel_obj and channel_obj.collection and not channel_obj.collection.file_format:
video_config = get_videos_downloader_config()
channel_obj.collection.file_format = video_config.file_name_format
save_channels_config.activate_switch()
session.commit()
with get_db_session(commit=True) as session:

View file

@ -16,7 +16,7 @@ from modules.videos.models import Video
from wrolpi import dates, flags
from wrolpi.captions import extract_captions
from wrolpi.common import ConfigFile, extract_domain, logger, \
escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config
escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config, trim_file_name
from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.downloader import Download, download_manager
@ -948,3 +948,85 @@ def browser_profile_to_yt_dlp_arg(profile: pathlib.Path) -> str:
"""Takes a Path and returns a string that can be used as a yt-dlp `--cookies-from-browser` argument."""
*_, browser, profile = str(profile).split('/')
return f'{browser}:{profile}'
def format_video_filename(
video: Video,
file_name_format: str = None,
) -> str:
"""Single source of truth for video filenames.
Formats a video filename using the configured file_name_format template.
Used by both the video downloader (initial download) and collection reorganization.
Args:
video: Video model with metadata (upload_date, uploader, title, source_id)
file_name_format: Optional override, defaults to config value
Returns:
Formatted filename (may include subdirectory path like "2025/01/video.mp4")
Example:
With format '%(upload_year)s/%(upload_month)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s':
Returns '2025/01/WROLPi_20250115_abc123_My Video Title.mp4'
"""
from modules.videos.downloader import convert_wrolpi_filename_format
config = get_videos_downloader_config()
template = file_name_format or config.file_name_format
# Get metadata from video
file_group = video.file_group
info_json = video.get_info_json() or {}
# Extract upload date info
upload_date = file_group.published_datetime
if not upload_date and info_json.get('upload_date'):
upload_date = dates.strpdate(info_json['upload_date'])
# Build variables dict matching yt-dlp's expected variables
# Use info_json if available, fall back to derived values
uploader = info_json.get('uploader') or info_json.get('channel') or ''
title = file_group.title or info_json.get('title') or info_json.get('fulltitle') or 'untitled'
source_id = video.source_id or info_json.get('id') or ''
# Get video file extension
video_path = video.video_path
ext = video_path.suffix.lstrip('.') if video_path else config.merge_output_format or 'mp4'
variables = dict(
uploader=escape_file_name(uploader) if uploader else '',
channel=escape_file_name(uploader) if uploader else '', # Alias
upload_date=upload_date.strftime('%Y%m%d') if upload_date else '',
id=source_id,
title=escape_file_name(title) if title else 'untitled',
ext=ext,
# WROLPi-specific variables for subdirectory organization
upload_year=str(upload_date.year) if upload_date else '',
upload_month=f'{upload_date.month:02d}' if upload_date else '',
)
# Convert WROLPi-specific variables to yt-dlp syntax then apply
# (This handles %(upload_year)s -> %(upload_date>%Y)s conversion)
converted_template = convert_wrolpi_filename_format(template)
# For the converted variables (like %(upload_date>%Y)s), we need to add them too
variables['upload_date>%Y'] = str(upload_date.year) if upload_date else ''
variables['upload_date>%m'] = f'{upload_date.month:02d}' if upload_date else ''
try:
result = converted_template % variables
# Trim the filename portion if it's too long (may include subdirectory paths)
if '/' in result:
# Has subdirectories - only trim the filename part
parts = result.rsplit('/', 1)
subdir = parts[0]
filename = trim_file_name(parts[1])
return f'{subdir}/{filename}'
else:
return trim_file_name(result)
except KeyError as e:
logger.error(f'Invalid variable in video file_name_format: {e}')
# Fallback to a simple format
fallback_title = escape_file_name(title) if title else 'untitled'
return trim_file_name(f'{fallback_title}.{ext}')

View file

@ -785,6 +785,9 @@ class Channel(ModelHelper, Base):
tag_name=self.tag_name,
url=self.url or None,
)
# Include file_format from the underlying Collection
if self.collection and self.collection.file_format:
config['file_format'] = self.collection.file_format
return config
@classmethod
@ -860,6 +863,7 @@ class Channel(ModelHelper, Base):
d['rss_url'] = self.get_rss_url()
d['directory'] = \
self.directory.relative_to(get_media_directory()) if self.directory else None
d['needs_reorganization'] = self.collection.needs_reorganization if self.collection else False
if with_statistics:
d['statistics'] = self.get_statistics()
if with_downloads:

View file

@ -280,3 +280,45 @@ class TestChannelsConfigEdgeCases:
# Directory should be relative, not absolute
assert not config['directory'].startswith('/'), \
f"Expected relative path but got absolute: {config['directory']}"
async def test_channel_config_view_includes_file_format(self, test_session, test_directory, async_client,
test_channels_config, channel_factory):
"""Channel.config_view() should include file_format from underlying Collection."""
channel = channel_factory()
channel.collection.file_format = '%(uploader)s_%(id)s_%(title)s.%(ext)s'
test_session.commit()
config = channel.config_view()
assert config['file_format'] == '%(uploader)s_%(id)s_%(title)s.%(ext)s'
async def test_channel_file_format_round_trip(self, test_session, test_directory, async_client,
test_channels_config, channel_factory):
"""File format should survive config dump and import cycle."""
config = get_channels_config()
config_path = config.get_file()
# Create channel with file_format
channel = channel_factory(name='Format Test')
channel.collection.file_format = '%(upload_date)s_%(title)s.%(ext)s'
test_session.commit()
# Dump to config
config.dump_config()
# Verify file_format in YAML
with open(config_path) as f:
config_data = yaml.safe_load(f)
channel_config = next(c for c in config_data['channels'] if c['name'] == 'Format Test')
assert channel_config['file_format'] == '%(upload_date)s_%(title)s.%(ext)s'
# Clear the file_format to simulate factory reset
channel.collection.file_format = None
test_session.commit()
# Re-import
config.initialize()
config.import_config()
# Verify file_format restored
test_session.refresh(channel)
assert channel.collection.file_format == '%(upload_date)s_%(title)s.%(ext)s'

View file

@ -8,7 +8,8 @@ from unittest import mock
import pytest
from modules.videos.downloader import VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader, preview_filename
get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
prepare_filename, convert_wrolpi_filename_format
from modules.videos.lib import get_videos_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
@ -573,6 +574,54 @@ def test_preview_filename_with_upload_year(test_directory, fake_now):
== '2000/WROLPi_20000101_The title of the video.mp4'
def test_preview_filename_matches_youtubedl(test_directory, fake_now):
"""Ensure our lightweight preview_filename() matches YoutubeDL's actual output.
This test validates that our simple string formatting produces identical
results to YoutubeDL's prepare_filename(). If yt-dlp changes behavior,
this test will catch it.
"""
from copy import deepcopy
from yt_dlp import YoutubeDL
from wrolpi.dates import now
test_formats = [
'%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
'%(upload_date)s_%(id)s_%(title)s.%(ext)s',
'%(upload_year)s/%(upload_month)s/%(title)s.%(ext)s',
'%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s',
'%(channel)s/%(id)s.%(ext)s',
]
# Sample entry used by both methods - must match preview_filename's variables
sample_date = now()
entry = dict(
uploader='WROLPi',
channel='WROLPi',
timestamp=int(sample_date.timestamp()),
upload_date=sample_date.strftime('%Y%m%d'),
id='Qz-FuenRylQ',
title='The title of the video',
ext='mp4',
description='A description of the video',
)
for fmt in test_formats:
# Get result from our lightweight implementation
lightweight_result = preview_filename(fmt)
# Get result from actual YoutubeDL
converted_format = convert_wrolpi_filename_format(fmt)
options = get_videos_downloader_config().yt_dlp_options
options = deepcopy(options)
options['outtmpl'] = converted_format
ydl = YoutubeDL(options)
ytdlp_result = prepare_filename(entry, ydl=ydl).lstrip('/')
assert lightweight_result == ytdlp_result, \
f"Mismatch for format '{fmt}':\n lightweight: {lightweight_result}\n ytdlp: {ytdlp_result}"
def test_normalize_video_file_names(test_directory, video_download_manager):
"""VideoDownloader renames caption files that do not share the video file's stem."""
downloader = VideoDownloader()

View file

@ -326,3 +326,32 @@ def test_video_location(test_session, video_factory, channel_factory, async_clie
channel = channel_factory(name='ChannelName')
video2 = video_factory(title='vid2', channel_id=channel.id)
assert video2.location == '/videos/channel/1/video/2'
def test_format_video_filename_without_upload_date(test_session, video_factory, test_videos_downloader_config, caplog):
"""Videos without upload_date should not cause KeyError when format uses upload_year."""
import logging
from modules.videos.lib import format_video_filename
# Create video without upload_date
video = video_factory(title='Test Video', with_video_file=True, with_info_json={'id': 'abc123'})
assert video.file_group.published_datetime is None
# Use a format that includes upload_year subdirectory
template = '%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
# Should not raise KeyError and should not log an error
with caplog.at_level(logging.ERROR):
result = format_video_filename(video, template)
# Should not have logged any errors (fallback was not used)
assert 'Invalid variable in video file_name_format' not in caplog.text
# Should return a properly formatted filename with the template structure
# (empty year and date components produce a path with subdirectory)
assert result.endswith('.mp4')
assert 'Test Video' in result
# The result should follow the template format, not the fallback format
# Template result: '{upload_year}/{uploader}_{upload_date}_{id}_{title}.mp4'
# With empty year/date: '/__Test Video_Test Video.mp4' (has a slash for subdirectory)
assert '/' in result # Confirms template was used (has subdirectory), not fallback

View file

@ -5,5 +5,5 @@ testpaths =
modules
asyncio_default_fixture_loop_scope = function
# Do not remove timeout - tests should not run forever
timeout = 20
timeout = 30
timeout_method = signal

View file

@ -30,4 +30,4 @@ srt==3.5.3
tqdm==4.67.1
vininfo[cli]==1.8.0
webvtt-py==0.5.1
yt-dlp==2025.12.08
yt-dlp==2026.01.29

View file

@ -79,11 +79,54 @@ CHROMIUM = which('chromium-browser', 'chromium',
'/usr/bin/chromium-browser', # rpi os
'/usr/bin/chromium', # debian
)
FIREFOX = which('firefox', 'firefox-esr',
'/usr/bin/firefox',
'/usr/bin/firefox-esr',
)
READABILITY_BIN = which('readability-extractor',
'/usr/bin/readability-extractor', # rpi os
'/usr/local/bin/readability-extractor', # debian
)
# Known browser definitions for SingleFile
KNOWN_BROWSERS = {
'chromium': {
'paths': ['chromium-browser', 'chromium', '/usr/bin/chromium-browser', '/usr/bin/chromium'],
'name': 'Chromium',
},
'firefox': {
'paths': ['firefox', 'firefox-esr', '/usr/bin/firefox', '/usr/bin/firefox-esr'],
'name': 'Firefox',
},
'brave': {
'paths': ['brave-browser', '/usr/bin/brave-browser'],
'name': 'Brave',
}
}
def get_installed_browsers() -> list:
"""
Detect browsers installed on the system that can be used with SingleFile.
Returns a list of dicts with:
- key: The browser identifier (e.g., 'chromium', 'firefox')
- name: Human-readable name (e.g., 'Chromium', 'Firefox')
- path: Absolute path to the executable
"""
browsers = []
for key, info in KNOWN_BROWSERS.items():
path = which(*info['paths'])
if path:
browsers.append({
'key': key,
'name': info['name'],
'path': str(path),
})
return browsers
# Videos
YT_DLP_BIN = which(
'/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location

View file

@ -266,3 +266,224 @@ async def search_collections_endpoint(request: Request, body: schema.CollectionS
'collections': collections,
'totals': {'collections': len(collections)}
})
@collection_bp.get('/<collection_id:int>/reorganize/preview')
@openapi.summary('Preview collection reorganization')
@openapi.response(HTTPStatus.OK, schema.ReorganizationPreviewResponse, description="Reorganization preview")
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
async def preview_reorganization_endpoint(request: Request, collection_id: int):
"""
Get preview with sample of files to be renamed.
Returns a preview of what would happen if the collection is reorganized.
This is useful for showing users what files would move before they commit.
Returns:
- collection_id: The collection ID
- collection_name: The collection name
- total_files: Total FileGroups in collection
- files_needing_move: Count needing reorganization
- sample_moves: ~10 example moves [{old_path, new_path}, ...]
- new_file_format: The format template that will be applied
- current_file_format: The format currently stored on collection
"""
from wrolpi.collections.reorganize import get_reorganization_preview
session = request.ctx.session
try:
# Use exact_count=True for individual collection preview to get accurate count
preview = get_reorganization_preview(collection_id, session, exact_count=True)
return json_response({
'collection_id': preview.collection_id,
'collection_name': preview.collection_name,
'total_files': preview.total_files,
'files_needing_move': preview.files_needing_move,
'sample_moves': preview.sample_moves,
'new_file_format': preview.new_file_format,
'current_file_format': preview.current_file_format,
})
except UnknownCollection as e:
return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND)
except ValueError as e:
return json_response({'error': str(e)}, status=HTTPStatus.BAD_REQUEST)
@collection_bp.post('/<collection_id:int>/reorganize')
@openapi.summary('Execute collection reorganization')
@openapi.response(HTTPStatus.OK, schema.ReorganizationExecuteResponse, description="Reorganization started")
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@wrol_mode_check
async def execute_reorganization_endpoint(request: Request, collection_id: int):
"""
Execute reorganization.
Moves files to match the current file_name_format configuration.
The operation runs in the background via FileWorker.
Returns:
- job_id: For tracking progress via the status endpoint
- message: Status message
"""
from wrolpi.collections.reorganize import execute_reorganization
session = request.ctx.session
try:
job_id = execute_reorganization(collection_id, session)
if job_id:
return json_response({
'job_id': job_id,
'message': 'Reorganization started'
})
else:
return json_response({
'job_id': '',
'message': 'No files need reorganization'
})
except UnknownCollection as e:
return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND)
except ValueError as e:
return json_response({'error': str(e)}, status=HTTPStatus.BAD_REQUEST)
@collection_bp.get('/<collection_id:int>/reorganize/status/<job_id:str>')
@openapi.summary('Get reorganization status')
@openapi.response(HTTPStatus.OK, schema.ReorganizationStatusResponse, description="Reorganization status")
async def get_reorganization_status_endpoint(_: Request, collection_id: int, job_id: str):
"""
Get status of in-flight reorganization.
Returns:
- status: 'pending' | 'running' | 'complete' | 'failed'
- total: Total files to move
- completed: Files moved so far
- percent: Completion percentage
- error: Error message if failed
"""
from wrolpi.collections.reorganize import get_reorganization_status
status = get_reorganization_status(job_id)
return json_response(status)
# ============================================================================
# Batch Reorganization Endpoints
# ============================================================================
@collection_bp.get('/reorganize/channels')
@openapi.summary('List channels needing batch reorganization')
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationListResponse,
description="List of channels needing reorganization")
async def list_channels_needing_reorganization_endpoint(request: Request):
"""
List all channels that need reorganization because their file_name_format
differs from the current config.
Returns:
- collections: List of channel info with sample moves
- total_collections: Total count needing reorganization
- total_files_needing_move: Sum of all files needing reorganization
- new_file_format: The config format that will be applied
"""
from wrolpi.collections.reorganize import get_collections_needing_reorganization
session = request.ctx.session
result = get_collections_needing_reorganization('channel', session)
return json_response(result)
@collection_bp.get('/reorganize/domains')
@openapi.summary('List domains needing batch reorganization')
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationListResponse,
description="List of domains needing reorganization")
async def list_domains_needing_reorganization_endpoint(request: Request):
"""
List all domains that need reorganization because their file_name_format
differs from the current config.
Returns:
- collections: List of domain info with sample moves
- total_collections: Total count needing reorganization
- total_files_needing_move: Sum of all files needing reorganization
- new_file_format: The config format that will be applied
"""
from wrolpi.collections.reorganize import get_collections_needing_reorganization
session = request.ctx.session
result = get_collections_needing_reorganization('domain', session)
return json_response(result)
@collection_bp.post('/reorganize/channels')
@openapi.summary('Execute batch reorganization of all channels')
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationExecuteResponse,
description="Batch reorganization started")
@wrol_mode_check
async def execute_batch_reorganization_channels_endpoint(request: Request):
"""
Execute batch reorganization of all channels that need it.
Processes channels sequentially. If any channel fails, the batch stops
and reports which channel failed.
Returns:
- batch_job_id: For tracking progress via the batch status endpoint
- message: Status message
- collection_count: Number of channels to process
"""
from wrolpi.collections.reorganize import execute_batch_reorganization
session = request.ctx.session
result = execute_batch_reorganization('channel', session)
return json_response(result)
@collection_bp.post('/reorganize/domains')
@openapi.summary('Execute batch reorganization of all domains')
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationExecuteResponse,
description="Batch reorganization started")
@wrol_mode_check
async def execute_batch_reorganization_domains_endpoint(request: Request):
"""
Execute batch reorganization of all domains that need it.
Processes domains sequentially. If any domain fails, the batch stops
and reports which domain failed.
Returns:
- batch_job_id: For tracking progress via the batch status endpoint
- message: Status message
- collection_count: Number of domains to process
"""
from wrolpi.collections.reorganize import execute_batch_reorganization
session = request.ctx.session
result = execute_batch_reorganization('domain', session)
return json_response(result)
@collection_bp.get('/reorganize/batch/status/<batch_job_id:str>')
@openapi.summary('Get batch reorganization status')
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationStatusResponse,
description="Batch reorganization status")
async def get_batch_reorganization_status_endpoint(_: Request, batch_job_id: str):
"""
Get status of a batch reorganization job.
Returns:
- status: 'pending' | 'running' | 'complete' | 'failed' | 'unknown'
- total_collections: Total collections in batch
- completed_collections: Number completed so far
- current_collection: Info about currently processing collection
- overall_percent: Overall completion percentage
- completed: List of completed collection info
- failed_collection: Info about failed collection (if any)
- error: Error message if failed
"""
from wrolpi.collections.reorganize import get_batch_reorganization_status
status = get_batch_reorganization_status(batch_job_id)
return json_response(status)

View file

@ -1,9 +1,15 @@
from wrolpi.errors import APIError
__all__ = ['UnknownCollection']
__all__ = ['UnknownCollection', 'ReorganizationConflict']
class UnknownCollection(APIError):
"""Cannot find Collection"""
code = 'UNKNOWN_COLLECTION'
summary = 'Cannot find Collection'
class ReorganizationConflict(APIError):
"""Reorganization would cause filename conflicts"""
code = 'REORGANIZATION_CONFLICT'
summary = 'Reorganization would cause filename conflicts'

View file

@ -90,6 +90,10 @@ class Collection(ModelHelper, Base):
tag_id = Column(Integer, ForeignKey('tag.id'))
tag = relationship('Tag', primaryjoin='Collection.tag_id==Tag.id')
# Stores the file_name_format used when files were last organized
# Used to detect when reorganization is needed after config changes
file_format = Column(String, nullable=True)
created_date = Column(DateTime, server_default=func.now(), nullable=False)
# Columns updated by triggers (similar to Channel)
@ -119,6 +123,36 @@ class Collection(ModelHelper, Base):
def tag_name(self) -> str | None:
return self.tag.name if self.tag else None
@property
def needs_reorganization(self) -> bool:
"""Returns True if collection's file_format differs from current config.
When a user changes the file_name_format in their config, existing files
remain in their old locations. This property detects that mismatch so the
UI can offer to reorganize files.
Returns True when:
- file_format is None (collection has never been synced to config)
- file_format differs from current config format
"""
current = self._get_current_file_format()
if not current:
return False
if not self.file_format:
# Never been reorganized - needs to sync to current config
return True
return current != self.file_format
def _get_current_file_format(self) -> str | None:
"""Get current file_name_format from config based on collection kind."""
if self.kind == 'channel':
from modules.videos.lib import get_videos_downloader_config
return get_videos_downloader_config().file_name_format
elif self.kind == 'domain':
from modules.archive.lib import get_archive_downloader_config
return get_archive_downloader_config().file_name_format
return None
@property
def location(self) -> str:
"""The href of the collection in the App."""
@ -175,6 +209,9 @@ class Collection(ModelHelper, Base):
config['tag_name'] = self.tag_name
if self.file_format:
config['file_format'] = self.file_format
# Include downloads if any exist
if self.downloads:
config['downloads'] = [
@ -251,6 +288,7 @@ class Collection(ModelHelper, Base):
description = data.get('description')
directory = data.get('directory')
tag_name = data.get('tag_name')
file_format = data.get('file_format')
kind = (data.get('kind') or 'channel').strip()
# Validate known kinds (forward-compatible: allow future values)
if kind not in {'channel', 'domain'}:
@ -280,6 +318,7 @@ class Collection(ModelHelper, Base):
collection.description = description
collection.directory = directory
collection.kind = kind
collection.file_format = file_format
if tag_name:
tag = Tag.get_by_name(session, tag_name)
@ -298,6 +337,7 @@ class Collection(ModelHelper, Base):
description=description,
directory=directory,
kind=kind,
file_format=file_format,
)
if tag_name:
@ -387,6 +427,7 @@ class Collection(ModelHelper, Base):
description = data.get('description')
directory = data.get('directory')
tag_name = data.get('tag_name')
file_format = data.get('file_format')
kind = (data.get('kind') or 'channel').strip()
# Validate known kinds
@ -421,6 +462,7 @@ class Collection(ModelHelper, Base):
collection.description = description
collection.directory = directory
collection.kind = kind
collection.file_format = file_format
if tag_name:
tag = tags_by_name.get(tag_name)
@ -439,6 +481,7 @@ class Collection(ModelHelper, Base):
description=description,
directory=directory,
kind=kind,
file_format=file_format,
)
if tag_name:
@ -824,6 +867,8 @@ class Collection(ModelHelper, Base):
'item_count': self.item_count,
'total_size': self.total_size,
'downloads': self.downloads,
'file_format': self.file_format,
'needs_reorganization': self.needs_reorganization,
}
def format_directory(self, tag_name: Optional[str]) -> pathlib.Path:
@ -898,13 +943,14 @@ class Collection(ModelHelper, Base):
Absolute path to the collection's directory
"""
if self.directory:
return get_media_directory() / self.directory
# Return absolute path (joining with media_directory handles both relative and absolute paths)
return self.directory if self.directory.is_absolute() else get_media_directory() / self.directory
# First use - format and save
tag_name = tag_name or (self.tag.name if self.tag else None)
destination = self.format_destination(tag_name)
self.directory = get_relative_to_media_directory(destination)
self.directory = destination # Store absolute path; MediaPathType handles conversion
session.commit()
# Trigger config save for persistence

View file

@ -0,0 +1,801 @@
"""
Collection Reorganization Module
Provides functionality to reorganize files within Collections (Domains/Channels)
when the file_name_format config changes.
Example: User changes archive format from '%(title)s.%(ext)s' to '%(download_year)s/%(title)s.%(ext)s'
- Before: 'My Article.html'
- After: '2025/My Article.html'
"""
import pathlib
from dataclasses import dataclass
from typing import List, Tuple, Optional
from sqlalchemy import func
from sqlalchemy.orm import Session, joinedload
from wrolpi.common import logger, get_media_directory
from wrolpi.db import get_db_session
from wrolpi.files.lib import glob_shared_stem
from .errors import UnknownCollection
from .models import Collection
logger = logger.getChild(__name__)
__all__ = [
'ReorganizationPreview',
'get_reorganization_preview',
'execute_reorganization',
'get_collections_needing_reorganization',
'execute_batch_reorganization',
'get_batch_reorganization_status',
]
@dataclass
class ReorganizationPreview:
"""Preview of a collection reorganization operation.
Note: For performance reasons, files_needing_move equals total_files.
We assume all files need moving since computing actual moves for every
file is expensive and would defeat the purpose of this preview.
"""
collection_id: int
collection_name: str
total_files: int # Total FileGroups in collection
files_needing_move: int # Equals total_files (assumed all need moving)
sample_moves: List[dict] # Sample of moves: [{old_path, new_path}, ...]
new_file_format: str # The format that will be applied
current_file_format: Optional[str] # The format currently stored on collection
def _get_format_required_variables(file_format: str) -> set:
"""Parse a format string and return the set of variable names it uses.
Example: '%(upload_year)s/%(title)s.%(ext)s' -> {'upload_year', 'title', 'ext'}
"""
import re
# Match %(variable_name)s patterns
return set(re.findall(r'%\(([^)]+)\)s', file_format))
def _video_has_required_metadata(video, file_format: str) -> bool:
"""Check if video has the metadata required by the format string.
Returns False only if the format requires a variable that the video cannot provide.
"""
required = _get_format_required_variables(file_format)
file_group = video.file_group
info_json = video.get_info_json() or {}
# Check each potentially-missing variable
# title and ext are always available (have fallbacks)
has_upload_date = bool(
file_group.published_datetime or info_json.get('upload_date')
)
has_source_id = bool(video.source_id or info_json.get('id'))
has_uploader = bool(info_json.get('uploader') or info_json.get('channel'))
# Map format variables to their metadata availability
# Variables that depend on upload_date
date_vars = {'upload_year', 'upload_month', 'upload_date', 'upload_date>%Y', 'upload_date>%m'}
# Variables that depend on source_id
id_vars = {'id'}
# Variables that depend on uploader
uploader_vars = {'uploader', 'channel'}
for var in required:
if var in date_vars and not has_upload_date:
return False
if var in id_vars and not has_source_id:
return False
if var in uploader_vars and not has_uploader:
return False
return True
def _compute_new_path_for_video(
video,
collection_directory: pathlib.Path,
file_format: str = None,
) -> Optional[pathlib.Path]:
"""Compute the new path for a video based on current config.
Args:
video: Video model instance
collection_directory: Base directory for the collection
file_format: Optional format string override, defaults to config value
Returns None if the video lacks required metadata for the configured format.
"""
from modules.videos.lib import format_video_filename, get_videos_downloader_config
if file_format is None:
config = get_videos_downloader_config()
file_format = config.file_name_format
if not _video_has_required_metadata(video, file_format):
logger.info(f'Skipping video {video.id}: missing metadata required by format "{file_format}"')
return None
try:
new_filename = format_video_filename(video, file_format)
return collection_directory / new_filename
except Exception as e:
logger.warning(f'Failed to compute new path for video {video.id}: {e}')
return None
def _archive_has_required_metadata(archive, file_format: str) -> bool:
"""Check if archive has the metadata required by the format string.
Returns False only if the format requires a variable that the archive cannot provide.
Archives have robust fallbacks for most variables (dates fallback to now(),
title to 'untitled'). Only domain can be empty when URL is missing.
"""
required = _get_format_required_variables(file_format)
file_group = archive.file_group
# Domain requires a URL
has_domain = bool(file_group.url)
# Domain-related variables
domain_vars = {'domain'}
for var in required:
if var in domain_vars and not has_domain:
return False
# All other variables (title, download_*, ext) always have fallbacks
return True
def _compute_new_path_for_archive(
archive,
collection_directory: pathlib.Path,
file_format: str = None,
) -> Optional[pathlib.Path]:
"""Compute the new path for an archive based on current config.
Args:
archive: Archive model instance
collection_directory: Base directory for the collection
file_format: Optional format string override, defaults to config value
Returns None if the archive lacks required metadata for the configured format.
"""
from modules.archive.lib import format_archive_filename_from_archive, get_archive_downloader_config
if file_format is None:
config = get_archive_downloader_config()
file_format = config.file_name_format
if not _archive_has_required_metadata(archive, file_format):
logger.info(f'Skipping archive {archive.id}: missing metadata required by format "{file_format}"')
return None
try:
new_filename = format_archive_filename_from_archive(archive, file_format)
return collection_directory / new_filename
except Exception as e:
logger.warning(f'Failed to compute new path for archive {archive.id}: {e}')
return None
def get_reorganization_preview(
collection_id: int,
session: Session = None,
sample_size: int = 10,
exact_count: bool = False,
) -> ReorganizationPreview:
"""Get preview with sample of files to be renamed.
Does NOT compute all moves by default - only samples for preview display.
Args:
collection_id: The collection to preview reorganization for
session: Database session (optional, will create one if not provided)
sample_size: Number of sample moves to include in preview
exact_count: If True, compute the actual count of files needing moves
(more expensive but accurate for individual collection preview)
Returns:
ReorganizationPreview with counts and sample moves
Raises:
UnknownCollection: If collection not found
"""
if session is None:
with get_db_session() as session:
return get_reorganization_preview(collection_id, session, sample_size, exact_count)
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
if not collection:
raise UnknownCollection(f"Collection with ID {collection_id} not found")
if not collection.directory:
raise ValueError(f"Collection '{collection.name}' has no directory. Cannot reorganize.")
collection_directory = pathlib.Path(collection.directory)
if not collection_directory.is_absolute():
collection_directory = get_media_directory() / collection_directory
# Get the current file format from config
current_config_format = collection._get_current_file_format()
sample_moves = []
files_needing_move = 0
total_files = 0
if collection.kind == 'channel':
from modules.videos.models import Video, Channel
# Get the channel for this collection
channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
if not channel:
return ReorganizationPreview(
collection_id=collection_id,
collection_name=collection.name,
total_files=0,
files_needing_move=0,
sample_moves=[],
new_file_format=current_config_format or '',
current_file_format=collection.file_format,
)
# Use COUNT for total_files (fast)
total_files = session.query(func.count(Video.id)).filter_by(channel_id=channel.id).scalar()
if exact_count:
# Compute actual count by building full move mappings
move_mappings = _build_move_mappings_for_channel(collection, session)
files_needing_move = len(move_mappings)
else:
# Assume all files need moving (computing actual count is expensive)
files_needing_move = total_files
# Only load limited videos for sample search
videos = session.query(Video).options(
joinedload(Video.file_group)
).filter_by(channel_id=channel.id).limit(10).all()
for video in videos:
if not video.file_group or not video.file_group.primary_path:
continue
current_path = pathlib.Path(video.file_group.primary_path)
new_path = _compute_new_path_for_video(video, collection_directory)
if new_path and current_path != new_path:
# Get relative paths for display
media_dir = get_media_directory()
try:
old_relative = str(current_path.relative_to(media_dir))
except ValueError:
old_relative = str(current_path)
try:
new_relative = str(new_path.relative_to(media_dir))
except ValueError:
new_relative = str(new_path)
sample_moves.append({
'old_path': old_relative,
'new_path': new_relative,
})
if len(sample_moves) >= sample_size:
break
elif collection.kind == 'domain':
from modules.archive.models import Archive
# Use COUNT for total_files (fast)
total_files = session.query(func.count(Archive.id)).filter_by(collection_id=collection_id).scalar()
if exact_count:
# Compute actual count by building full move mappings
move_mappings = _build_move_mappings_for_domain(collection, session)
files_needing_move = len(move_mappings)
else:
# Assume all files need moving (computing actual count is expensive)
files_needing_move = total_files
# Only load limited archives for sample search
archives = session.query(Archive).options(
joinedload(Archive.file_group)
).filter_by(collection_id=collection_id).limit(10).all()
for archive in archives:
if not archive.file_group or not archive.file_group.primary_path:
continue
current_path = pathlib.Path(archive.file_group.primary_path)
new_path = _compute_new_path_for_archive(archive, collection_directory)
if new_path and current_path != new_path:
# Get relative paths for display
media_dir = get_media_directory()
try:
old_relative = str(current_path.relative_to(media_dir))
except ValueError:
old_relative = str(current_path)
try:
new_relative = str(new_path.relative_to(media_dir))
except ValueError:
new_relative = str(new_path)
sample_moves.append({
'old_path': old_relative,
'new_path': new_relative,
})
if len(sample_moves) >= sample_size:
break
return ReorganizationPreview(
collection_id=collection_id,
collection_name=collection.name,
total_files=total_files,
files_needing_move=files_needing_move,
sample_moves=sample_moves,
new_file_format=current_config_format or '',
current_file_format=collection.file_format,
)
def _build_move_mappings_for_channel(
collection: Collection,
session: Session,
) -> List[Tuple[pathlib.Path, pathlib.Path]]:
"""Build move mappings for a channel collection."""
from modules.videos.models import Video, Channel
collection_directory = pathlib.Path(collection.directory)
if not collection_directory.is_absolute():
collection_directory = get_media_directory() / collection_directory
channel = session.query(Channel).filter_by(collection_id=collection.id).one_or_none()
if not channel:
return []
move_mappings = []
videos = session.query(Video).options(
joinedload(Video.file_group)
).filter_by(channel_id=channel.id).all()
for video in videos:
if not video.file_group or not video.file_group.primary_path:
continue
current_path = pathlib.Path(video.file_group.primary_path)
new_path = _compute_new_path_for_video(video, collection_directory)
if new_path and current_path != new_path:
move_mappings.append((current_path, new_path))
return move_mappings
def _build_move_mappings_for_domain(
collection: Collection,
session: Session,
) -> List[Tuple[pathlib.Path, pathlib.Path]]:
"""Build move mappings for a domain collection."""
from modules.archive.models import Archive
collection_directory = pathlib.Path(collection.directory)
if not collection_directory.is_absolute():
collection_directory = get_media_directory() / collection_directory
move_mappings = []
archives = session.query(Archive).options(
joinedload(Archive.file_group)
).filter_by(collection_id=collection.id).all()
for archive in archives:
if not archive.file_group or not archive.file_group.primary_path:
continue
current_path = pathlib.Path(archive.file_group.primary_path)
new_path = _compute_new_path_for_archive(archive, collection_directory)
if new_path and current_path != new_path:
move_mappings.append((current_path, new_path))
return move_mappings
def _check_for_destination_conflicts(
move_mappings: List[Tuple[pathlib.Path, pathlib.Path]],
collection_name: str,
) -> None:
"""Check for duplicate destination paths in move mappings.
Raises:
ReorganizationConflict: If two or more files would move to the same destination
"""
from wrolpi.collections.errors import ReorganizationConflict
dest_to_sources = {}
for source, dest in move_mappings:
if dest in dest_to_sources:
dest_to_sources[dest].append(source)
else:
dest_to_sources[dest] = [source]
conflicts = {dest: sources for dest, sources in dest_to_sources.items() if len(sources) > 1}
if conflicts:
conflict_details = []
for dest, sources in conflicts.items():
source_names = [str(s.name) for s in sources]
conflict_details.append(f" {dest.name} <- {', '.join(source_names)}")
raise ReorganizationConflict(
f"Reorganization of '{collection_name}' would cause {len(conflicts)} filename conflict(s):\n" +
"\n".join(conflict_details)
)
def execute_reorganization(
collection_id: int,
session: Session = None,
) -> str:
"""Execute reorganization, returns job_id for tracking.
Iterates ALL FileGroups and queues moves to FileWorker.
Args:
collection_id: The collection to reorganize
session: Database session (optional)
Returns:
job_id for tracking progress via FileWorker status
Raises:
UnknownCollection: If collection not found
"""
from wrolpi.files.worker import file_worker
if session is None:
with get_db_session(commit=True) as session:
return execute_reorganization(collection_id, session)
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
if not collection:
raise UnknownCollection(f"Collection with ID {collection_id} not found")
if not collection.directory:
raise ValueError(f"Collection '{collection.name}' has no directory. Cannot reorganize.")
# Build move mappings based on collection kind
if collection.kind == 'channel':
move_mappings = _build_move_mappings_for_channel(collection, session)
elif collection.kind == 'domain':
move_mappings = _build_move_mappings_for_domain(collection, session)
else:
raise ValueError(f"Unsupported collection kind: {collection.kind}")
# Check for filename conflicts before proceeding
_check_for_destination_conflicts(move_mappings, collection.name)
if not move_mappings:
logger.info(f'No files need reorganization for collection {collection.name}')
# Still update the file_format to current config
current_format = collection._get_current_file_format()
if current_format:
collection.file_format = current_format
session.commit()
# Trigger config save
if collection.kind == 'domain':
from modules.archive.lib import save_domains_config
save_domains_config.activate_switch()
elif collection.kind == 'channel':
from modules.videos.lib import save_channels_config
save_channels_config.activate_switch()
return ''
logger.info(f'Queueing reorganization for collection {collection.name}: {len(move_mappings)} files')
# Get the current file format to update AFTER successful completion
# (deferred update enables retry on failure - file_format only updates on success)
pending_file_format = collection._get_current_file_format()
# Queue the reorganization with FileWorker
# The pending_file_format will be set by handle_reorganize after successful completion
job_id = file_worker.queue_reorganize(
move_mappings,
collection_id=collection_id,
collection_kind=collection.kind,
pending_file_format=pending_file_format,
)
return job_id
def get_reorganization_status(job_id: str) -> dict:
"""Get status of an in-flight reorganization.
Args:
job_id: The job ID returned by execute_reorganization
Returns:
Dict with status information:
- status: 'pending' | 'running' | 'complete' | 'failed'
- total: Total files to move
- completed: Files moved so far
- percent: Completion percentage
- error: Error message if failed
"""
from wrolpi.files.worker import file_worker
try:
job_status = file_worker.get_job_status(job_id)
except RuntimeError:
return {'status': 'unknown', 'error': f'Job {job_id} not found'}
# Get current worker status if this job is running
worker_status = file_worker.status
if job_status == 'complete':
return {
'status': 'complete',
'total': worker_status.get('operation_total', 0),
'completed': worker_status.get('operation_total', 0),
'percent': 100,
'error': None,
}
elif job_status == 'pending':
return {
'status': 'pending',
'total': 0,
'completed': 0,
'percent': 0,
'error': None,
}
else:
# Job is running
return {
'status': worker_status.get('status', 'running'),
'total': worker_status.get('operation_total', 0),
'completed': worker_status.get('operation_processed', 0),
'percent': worker_status.get('operation_percent', 0),
'error': worker_status.get('error'),
}
# ============================================================================
# Batch Reorganization Functions
# ============================================================================
def get_collections_needing_reorganization(
kind: str,
session: Session = None,
sample_size: int = 1,
) -> dict:
"""Get all collections of a kind that need reorganization with sample previews.
Args:
kind: 'channel' or 'domain'
session: Database session (optional)
sample_size: Number of sample moves per collection (default 1)
Returns:
Dict with:
- collections: List of collection info dicts
- total_collections: Total count needing reorganization
- total_files_needing_move: Sum of all files needing reorganization
- new_file_format: The config format that will be applied
"""
if session is None:
with get_db_session() as session:
return get_collections_needing_reorganization(kind, session, sample_size)
# Query all collections of this kind with directories
collections = session.query(Collection).filter(
Collection.kind == kind,
Collection.directory.isnot(None),
).all()
# Filter to only those needing reorganization
collections_needing_reorg = [c for c in collections if c.needs_reorganization]
# Get current config format
current_format = ''
if kind == 'channel':
from modules.videos.lib import get_videos_downloader_config
current_format = get_videos_downloader_config().file_name_format
elif kind == 'domain':
from modules.archive.lib import get_archive_downloader_config
current_format = get_archive_downloader_config().file_name_format
result_collections = []
total_files_needing_move = 0
for collection in collections_needing_reorg:
# Get preview for each collection
try:
preview = get_reorganization_preview(collection.id, session, sample_size=sample_size)
sample_move = preview.sample_moves[0] if preview.sample_moves else None
result_collections.append({
'collection_id': collection.id,
'collection_name': collection.name,
'total_files': preview.total_files,
'files_needing_move': preview.files_needing_move,
'sample_move': sample_move,
})
total_files_needing_move += preview.files_needing_move
except Exception as e:
logger.warning(f'Failed to get preview for collection {collection.id}: {e}')
result_collections.append({
'collection_id': collection.id,
'collection_name': collection.name,
'total_files': 0,
'files_needing_move': 0,
'sample_move': None,
})
return {
'collections': result_collections,
'total_collections': len(result_collections),
'total_files_needing_move': total_files_needing_move,
'new_file_format': current_format,
}
def execute_batch_reorganization(kind: str, session: Session = None) -> dict:
"""Execute batch reorganization for all collections of a kind that need it.
This queues a batch reorganization job that processes collections sequentially.
Each collection is reorganized one at a time to avoid system overload.
If any collection fails, the batch stops and reports which collection failed.
Args:
kind: 'channel' or 'domain'
session: Database session (optional)
Returns:
Dict with:
- batch_job_id: Job ID for tracking progress
- message: Status message
- collection_count: Number of collections to process
"""
from wrolpi.files.worker import file_worker
if session is None:
with get_db_session() as session:
return execute_batch_reorganization(kind, session)
# Get collections needing reorganization
reorg_info = get_collections_needing_reorganization(kind, session, sample_size=0)
collections_to_process = reorg_info['collections']
if not collections_to_process:
logger.info(f'No {kind} collections need reorganization')
return {
'batch_job_id': '',
'message': f'No {kind} collections need reorganization',
'collection_count': 0,
}
# Extract collection IDs for the batch job
collection_ids = [c['collection_id'] for c in collections_to_process]
logger.info(f'Queueing batch reorganization for {len(collection_ids)} {kind} collections')
# Queue the batch reorganization
batch_job_id = file_worker.queue_batch_reorganize(collection_ids, kind)
return {
'batch_job_id': batch_job_id,
'message': 'Batch reorganization started',
'collection_count': len(collection_ids),
}
def get_batch_reorganization_status(batch_job_id: str) -> dict:
"""Get status of a batch reorganization job.
Args:
batch_job_id: The batch job ID returned by execute_batch_reorganization
Returns:
Dict with:
- status: 'pending' | 'running' | 'complete' | 'failed' | 'unknown'
- total_collections: Total collections in batch
- completed_collections: Number completed so far
- current_collection: Info about currently processing collection (or None)
- overall_percent: Overall completion percentage
- completed: List of completed collection info
- failed_collection: Info about failed collection (or None)
- error: Error message if failed
"""
from wrolpi.files.worker import file_worker
try:
job_status = file_worker.get_job_status(batch_job_id)
except RuntimeError:
return {
'status': 'unknown',
'error': f'Batch job {batch_job_id} not found',
'total_collections': 0,
'completed_collections': 0,
'current_collection': None,
'overall_percent': 0,
'completed': [],
'failed_collection': None,
}
# Get batch-specific status from worker
worker_status = file_worker.status
batch_status = worker_status.get('batch_status', {})
if job_status == 'complete':
completed_list = batch_status.get('completed', [])
return {
'status': 'complete',
'total_collections': batch_status.get('total_collections', 0),
'completed_collections': len(completed_list),
'current_collection': None,
'overall_percent': 100,
'completed': completed_list,
'failed_collection': None,
'error': None,
}
elif job_status == 'pending':
return {
'status': 'pending',
'total_collections': batch_status.get('total_collections', 0),
'completed_collections': 0,
'current_collection': None,
'overall_percent': 0,
'completed': [],
'failed_collection': None,
'error': None,
}
elif job_status == 'failed':
completed_list = batch_status.get('completed', [])
return {
'status': 'failed',
'total_collections': batch_status.get('total_collections', 0),
'completed_collections': len(completed_list),
'current_collection': None,
'overall_percent': batch_status.get('overall_percent', 0),
'completed': completed_list,
'failed_collection': batch_status.get('failed_collection'),
'error': batch_status.get('error'),
}
else:
# Job is running
completed_list = batch_status.get('completed', [])
current = batch_status.get('current_collection')
total = batch_status.get('total_collections', 1)
# Get fresh file-level progress from worker status (not from batch_status which may be stale)
current_percent = worker_status.get('operation_percent', 0)
# Update current collection dict with fresh values if it exists
if current:
current = dict(current) # Copy to avoid mutating the original
current['percent'] = current_percent
current['completed'] = worker_status.get('operation_processed', 0)
current['total'] = worker_status.get('operation_total', 0)
# Calculate overall percent: (completed + current_progress) / total
completed_count = len(completed_list)
overall_percent = int(((completed_count + current_percent / 100) / total) * 100) if total > 0 else 0
return {
'status': 'running',
'total_collections': total,
'completed_collections': completed_count,
'current_collection': current,
'overall_percent': overall_percent,
'completed': completed_list,
'failed_collection': None,
'error': None,
}

View file

@ -52,3 +52,88 @@ class CollectionTagInfoResponse:
suggested_directory: str
conflict: bool
conflict_message: Optional[str] = None
@dataclass
class ReorganizationPreviewResponse:
"""Response for reorganization preview."""
collection_id: int
collection_name: str
total_files: int
files_needing_move: int
sample_moves: List[dict]
new_file_format: str
current_file_format: Optional[str] = None
@dataclass
class ReorganizationExecuteResponse:
"""Response for reorganization execution."""
job_id: str
message: str
@dataclass
class ReorganizationStatusResponse:
"""Response for reorganization status."""
status: str
total: int
completed: int
percent: int
error: Optional[str] = None
# ============================================================================
# Batch Reorganization Schemas
# ============================================================================
@dataclass
class BatchCollectionInfo:
"""Info about a single collection in batch reorganization."""
collection_id: int
collection_name: str
total_files: int
files_needing_move: int
sample_move: Optional[dict] = None
@dataclass
class BatchReorganizationListResponse:
"""Response for listing collections needing batch reorganization."""
collections: List[dict]
total_collections: int
total_files_needing_move: int
new_file_format: str
@dataclass
class BatchReorganizationExecuteResponse:
"""Response for executing batch reorganization."""
batch_job_id: str
message: str
collection_count: int
@dataclass
class BatchCollectionProgress:
"""Progress info for a single collection in batch reorganization."""
id: int
name: str
status: str
total: int = 0
completed: int = 0
percent: int = 0
@dataclass
class BatchReorganizationStatusResponse:
"""Response for batch reorganization status."""
status: str
total_collections: int
completed_collections: int
current_collection: Optional[dict]
overall_percent: int
completed: List[dict]
failed_collection: Optional[dict]
error: Optional[str] = None

File diff suppressed because it is too large Load diff

View file

@ -1404,27 +1404,45 @@ def escape_file_name(name: str) -> str:
return name.strip()
# Maximum length is probably 255, but we need more length for large suffixes like `.readability.json`, and temporary
# downloading suffixes from yt-dlp.
MAXIMUM_FILE_LENGTH = 140
# Maximum bytes for a filename. Linux ext4/xfs limit is 255 bytes.
# We use 200 to leave room for FileGroup suffix variations (e.g., .readability.html is 17 bytes vs .html at 5 bytes).
MAXIMUM_FILE_BYTES = 200
def trim_file_name(path: str | pathlib.Path) -> str | pathlib.Path:
"""Shorten the file name only if it is longer than the file system supports. Trim from the end of the name until
the name is short enough (preserving any suffix)."""
the name is short enough (preserving any suffix).
Uses BYTE length (not character length) because UTF-8 multi-byte characters (e.g., Devanagari, Chinese)
can exceed the 255 byte filesystem limit even with fewer than 140 characters.
"""
name_type = pathlib.Path if isinstance(path, pathlib.Path) else str
parent = path.parent if isinstance(path, pathlib.Path) else '/'.join(i for i in path.split('/')[:-1])
path = path.name if isinstance(path, pathlib.Path) else path
filename = path.name if isinstance(path, pathlib.Path) else path.split('/')[-1] if '/' in path else path
if len(path) < MAXIMUM_FILE_LENGTH:
return name_type(path)
filename_bytes = len(filename.encode('utf-8'))
if filename_bytes <= MAXIMUM_FILE_BYTES:
# No trimming needed, but preserve parent directory if present
if parent:
if name_type == pathlib.Path:
return parent / filename
return f'{parent}/{filename}'
return name_type(filename)
# Don't trim the filename to exactly 256 characters.
# Trim based on byte length, not character length.
# This is because a FileGroup will have varying filename lengths.
from wrolpi.files.lib import split_path_stem_and_suffix
stem, suffix = split_path_stem_and_suffix(path)
excess = MAXIMUM_FILE_LENGTH - len(suffix)
new_name = stem[:excess].strip() + suffix
stem, suffix = split_path_stem_and_suffix(filename)
suffix_bytes = len(suffix.encode('utf-8'))
target_stem_bytes = MAXIMUM_FILE_BYTES - suffix_bytes
# Trim characters from end of stem until byte length fits
trimmed_stem = stem
while len(trimmed_stem.encode('utf-8')) > target_stem_bytes and trimmed_stem:
trimmed_stem = trimmed_stem[:-1]
trimmed_stem = trimmed_stem.strip()
new_name = trimmed_stem + suffix
if parent:
if name_type == pathlib.Path:
return parent / new_name
@ -1573,7 +1591,22 @@ def register_modeler(modeler: callable):
return modeler
async def apply_modelers():
async def apply_modelers(progress_callback: callable = None):
"""Run all registered modelers.
Args:
progress_callback: Optional callback(processed, total) called after each modeler completes.
"""
from wrolpi.db import get_db_session
from wrolpi.files.models import FileGroup
# Get initial count of unindexed files for progress tracking
initial_unindexed = 0
if progress_callback:
with get_db_session() as session:
initial_unindexed = session.query(FileGroup).filter(FileGroup.indexed != True).count()
progress_callback(0, initial_unindexed)
for modeler in modelers:
logger_.info(f'Applying modeler {modeler.__name__}')
try:
@ -1582,6 +1615,14 @@ async def apply_modelers():
logger_.error(f'Modeler {modeler.__name__} raised error', exc_info=e)
if PYTEST:
raise
# Update progress after each modeler
if progress_callback and initial_unindexed > 0:
with get_db_session() as session:
remaining = session.query(FileGroup).filter(FileGroup.indexed != True).count()
processed = initial_unindexed - remaining
progress_callback(processed, initial_unindexed)
# Sleep to catch cancel.
await asyncio.sleep(0)

View file

@ -12,7 +12,7 @@ import subprocess
import urllib.parse
from itertools import zip_longest
from pathlib import Path
from typing import List, Tuple, Union, Dict, Generator, Iterable, Set
from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set
import cachetools.func
import psycopg2
@ -724,11 +724,23 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
curs.execute(stmt)
async def apply_indexers():
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function."""
async def apply_indexers(progress_callback: Callable[[int, int], None] = None):
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function.
Args:
progress_callback: Optional callback(processed, total) called after each batch.
"""
from wrolpi.files.models import FileGroup
refresh_logger.info('Applying indexers')
# Get initial count for progress tracking
total_to_index = 0
total_indexed = 0
if progress_callback:
with get_db_session() as session:
total_to_index = session.query(FileGroup).filter(FileGroup.indexed != True).count()
progress_callback(0, total_to_index)
while True:
# Continually query for Files that have not been indexed.
with get_db_session(commit=False) as session:
@ -809,6 +821,11 @@ async def apply_indexers():
if file_group:
refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}')
# Update progress
total_indexed += processed
if progress_callback and total_to_index > 0:
progress_callback(total_indexed, total_to_index)
if processed < 20:
# Processed less than the limit, don't do the next query.
break
@ -1365,6 +1382,51 @@ def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path,
curs.execute(sql)
def _json_serial(obj):
"""JSON serializer for objects not serializable by default json code."""
import datetime
if isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
raise TypeError(f"Type {type(obj)} not serializable")
def _bulk_update_file_groups_reorganize(updates: List[dict]):
"""Batch UPDATE FileGroups for reorganization using raw SQL.
Unlike _bulk_update_file_groups_db, this also updates the files and data JSON fields
since reorganization changes filenames, not just directories.
Args:
updates: List of dicts with keys: id, directory, primary_path, files, data
"""
if not updates:
return
with get_db_curs(commit=True) as curs:
for update in updates:
# Don't change indexed - reorganization only moves files, content unchanged
curs.execute('''
UPDATE file_group
SET directory = %s,
primary_path = %s,
files = %s,
data = %s
WHERE id = %s
''', (
update['directory'],
update['primary_path'],
json.dumps(update['files'], default=_json_serial),
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
update['id'],
))
# Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates.
# This is necessary because get_db_curs uses the same connection as get_db_session
# during tests, but SQLAlchemy's identity map doesn't know about raw SQL updates.
with get_db_session() as session:
session.expire_all()
def delete_directory(directory: pathlib.Path, recursive: bool = False):
"""Remove a directory, remove it's Directory record.

View file

@ -1360,3 +1360,51 @@ async def test_rename_non_primary_file_renames_filegroup(async_client, test_sess
assert 'renamed.mp4' in file_names
assert 'renamed.srt' in file_names
assert len(file_group.files) == 2
def test_bulk_update_file_groups_reorganize_handles_datetime(test_session, test_directory):
"""_bulk_update_file_groups_reorganize should handle datetime objects in files JSON.
The files JSON field can contain modification_datetime as a datetime object.
The function must serialize these correctly to avoid JSON serialization errors.
"""
from datetime import datetime, timezone
from wrolpi.files.lib import _bulk_update_file_groups_reorganize
# Create a test file and FileGroup
test_file = test_directory / 'test_video.mp4'
test_file.touch()
fg = FileGroup()
fg.directory = test_directory
fg.primary_path = test_file
fg.files = [{'path': 'test_video.mp4', 'mimetype': 'video/mp4'}]
test_session.add(fg)
test_session.commit()
fg_id = fg.id
# Prepare update with datetime object in files (simulating what handle_reorganize does)
new_files = [{
'path': 'test_video.mp4',
'mimetype': 'video/mp4',
'modification_datetime': datetime.now(timezone.utc), # datetime object, not string
}]
updates = [{
'id': fg_id,
'directory': str(test_directory),
'primary_path': str(test_file),
'files': new_files,
}]
# This should NOT raise "Object of type datetime is not JSON serializable"
_bulk_update_file_groups_reorganize(updates)
# Verify the update was applied
test_session.expire_all()
fg = test_session.query(FileGroup).filter_by(id=fg_id).one()
assert len(fg.files) == 1
assert fg.files[0]['path'] == 'test_video.mp4'
# modification_datetime should be serialized as ISO string
assert 'modification_datetime' in fg.files[0]

View file

@ -302,10 +302,11 @@ async def test_compare_file_groups_ignores_outside_root(test_session, test_direc
@pytest.mark.asyncio
async def test_compare_file_groups_cancelable(test_session, test_directory, make_files_structure):
"""compare_file_groups can be canceled during long operations."""
"""compare_file_groups can be canceled during long operations and reports progress via callback."""
# Create many files to ensure scan takes time
make_files_structure([f'docs/file{i}.txt' for i in range(100)])
# Test cancellation
task = asyncio.create_task(compare_file_groups(test_directory))
await asyncio.sleep(0) # Let task start
task.cancel()
@ -313,6 +314,13 @@ async def test_compare_file_groups_cancelable(test_session, test_directory, make
with pytest.raises(asyncio.CancelledError):
await task
# Test progress callback with small batch size
progress_counts = []
await compare_file_groups(test_directory, batch_size=10, progress_callback=progress_counts.append)
# Should have been called multiple times with increasing counts
assert len(progress_counts) >= 1
assert progress_counts == sorted(progress_counts) # Counts should be increasing
@pytest.mark.asyncio
async def test_count_files_empty(test_directory):

View file

@ -30,6 +30,7 @@ from wrolpi.files.lib import (
split_path_stem_and_suffix, _upsert_files, get_unique_files_by_stem, glob_shared_stem,
group_files_by_stem, get_primary_file, delete_directory, apply_indexers,
_move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE,
_bulk_update_file_groups_reorganize,
)
logger = logger.getChild(__name__)
@ -274,7 +275,11 @@ def _insert_fs_batch(session, batch: list):
)
async def compare_file_groups(root: Path = None, batch_size: int = 10000) -> FileComparisonResult:
async def compare_file_groups(
root: Path = None,
batch_size: int = 10000,
progress_callback: Callable[[int], None] = None,
) -> FileComparisonResult:
"""
Compare FileGroups in DB to filesystem. Memory efficient for Raspberry Pi.
@ -317,6 +322,8 @@ async def compare_file_groups(root: Path = None, batch_size: int = 10000) -> Fil
_insert_fs_batch(session, batch)
file_count += len(batch)
batch = []
if progress_callback:
progress_callback(file_count)
await asyncio.sleep(0) # Cancellation yield point
finally:
await path_gen.aclose() # Ensure subprocess is killed on cancellation
@ -691,6 +698,8 @@ class FileTaskType(str, Enum):
move = auto() # Move files and directories to a new location, file names are not changed.
rename = auto() # Rename files, requires deeper updating of FileGroup.files and FileGroup.data.
tag = auto() # Add TagFile records for files.
reorganize = auto() # Reorganize files with arbitrary source->dest mappings for collection reorganization.
batch_reorganize = auto() # Batch reorganize multiple collections sequentially.
@dataclass
@ -708,6 +717,15 @@ class FileTask:
next_task_type: FileTaskType = None
job_id: str = None # Unique ID for tracking completion
expand_stems: bool = True # Whether to expand files to their FileGroup stem-mates
# For reorganize tasks: list of (source_path, dest_path) tuples
move_mappings: list[tuple[pathlib.Path, pathlib.Path]] = None
collection_id: int = None # Collection being reorganized
collection_kind: str = None # 'channel' or 'domain' for reorganize tasks
# For batch_reorganize tasks
collection_ids: list[int] = None # List of collection IDs to process
batch_kind: str = None # 'channel' or 'domain'
# For deferred file_format update (only updated after successful completion)
pending_file_format: str = None
class FileWorker:
@ -844,6 +862,75 @@ class FileWorker:
self.public_queue.put_nowait(task)
return job_id
def queue_reorganize(
self,
move_mappings: list[tuple[pathlib.Path, pathlib.Path]],
collection_id: int = None,
collection_kind: str = None,
pending_file_format: str = None,
) -> str:
"""Queue a reorganize task for collection file reorganization.
Unlike queue_move which moves all files to a single destination,
queue_reorganize handles arbitrary source->destination mappings
where each file can move to a different location.
Args:
move_mappings: List of (source_path, dest_path) tuples
collection_id: Optional collection ID being reorganized (for status tracking)
collection_kind: 'channel' or 'domain' (for status tracking / navbar navigation)
pending_file_format: File format to set on collection after successful completion
(deferred update enables retry on failure)
Returns:
Job ID for tracking completion
"""
import uuid
job_id = f'reorganize-{uuid.uuid4().hex[:8]}'
# Paths list is empty for reorganize tasks - we use move_mappings instead
task = FileTask(
FileTaskType.reorganize,
paths=[],
move_mappings=move_mappings,
collection_id=collection_id,
collection_kind=collection_kind,
job_id=job_id,
pending_file_format=pending_file_format,
)
self._set_job_status(job_id, 'pending')
self.public_queue.put_nowait(task)
return job_id
def queue_batch_reorganize(
self,
collection_ids: list[int],
kind: str,
) -> str:
"""Queue a batch reorganize task for multiple collections.
Processes collections sequentially, stopping on first failure.
Tracks overall progress and per-collection progress.
Args:
collection_ids: List of collection IDs to reorganize
kind: 'channel' or 'domain'
Returns:
Batch job ID for tracking completion
"""
import uuid
job_id = f'batch-reorganize-{uuid.uuid4().hex[:8]}'
task = FileTask(
FileTaskType.batch_reorganize,
paths=[],
collection_ids=collection_ids,
batch_kind=kind,
job_id=job_id,
)
self._set_job_status(job_id, 'pending')
self.public_queue.put_nowait(task)
return job_id
async def refresh_sync(self, paths: list[pathlib.Path]):
"""Synchronously refresh specific files. Use sparingly - prefer queue_refresh."""
if not paths:
@ -864,17 +951,32 @@ class FileWorker:
try:
task: FileTask = self.private_queue.get_nowait()
except asyncio.QueueEmpty:
# No file tasks to perform. Sleep to catch cancel.
# No tasks - clear flag if set and both queues are empty
if flags.file_worker_busy.is_set() and self.public_queue.qsize() == 0:
flags.file_worker_busy.clear()
await asyncio.sleep(0)
return
match task.task_type:
case FileTaskType.count:
await self.handle_count(task)
case FileTaskType.refresh:
await self.handle_refresh(task)
case FileTaskType.move:
await self.handle_move(task)
# Set flag before processing any task (if not already set)
if not flags.file_worker_busy.is_set():
flags.file_worker_busy.set()
try:
match task.task_type:
case FileTaskType.count:
await self.handle_count(task)
case FileTaskType.refresh:
await self.handle_refresh(task)
case FileTaskType.move:
await self.handle_move(task)
case FileTaskType.reorganize:
await self.handle_reorganize(task)
case FileTaskType.batch_reorganize:
await self.handle_batch_reorganize(task)
finally:
# Clear flag if both queues are now empty
if self.private_queue.qsize() == 0 and self.public_queue.qsize() == 0:
flags.file_worker_busy.clear()
async def handle_count(self, task: FileTask):
"""Count files in the task's paths and chain to next task if specified."""
@ -1006,8 +1108,12 @@ class FileWorker:
)
logger.info(f'Comparing {task.count} files in {len(dir_paths)} directories')
def on_compare_progress(count: int):
percent = int((count / task.count) * 100) if task.count > 0 else 0
self.update_status(operation_processed=count, operation_percent=percent)
root = dir_paths[0] if len(dir_paths) == 1 else None
dir_result = await compare_file_groups(root=root)
dir_result = await compare_file_groups(root=root, progress_callback=on_compare_progress)
if is_global_refresh:
Events.send_global_refresh_discovery_completed()
@ -1661,15 +1767,23 @@ class FileWorker:
"""
from wrolpi.tags import save_tags_config
self.update_status(status='modeling')
def on_modeling_progress(processed: int, total: int):
percent = int((processed / total) * 100) if total > 0 else 0
self.update_status(operation_total=total, operation_processed=processed, operation_percent=percent)
self.update_status(status='modeling', operation_total=0, operation_processed=0, operation_percent=0)
with flags.file_worker_modeling:
await apply_modelers()
await apply_modelers(progress_callback=on_modeling_progress)
if is_global_refresh:
Events.send_global_refresh_modeling_completed()
self.update_status(status='indexing')
def on_indexing_progress(processed: int, total: int):
percent = int((processed / total) * 100) if total > 0 else 0
self.update_status(operation_total=total, operation_processed=processed, operation_percent=percent)
self.update_status(status='indexing', operation_total=0, operation_processed=0, operation_percent=0)
with flags.file_worker_indexing:
await apply_indexers()
await apply_indexers(progress_callback=on_indexing_progress)
if is_global_refresh:
Events.send_global_refresh_indexing_completed()
@ -1763,27 +1877,26 @@ class FileWorker:
plan = {}
try:
with flags.file_worker_busy:
destination.mkdir(parents=True, exist_ok=True)
destination.mkdir(parents=True, exist_ok=True)
# Build the move plan using bulk SQL operations
plan, old_directories = await build_move_plan_bulk(sources, destination)
# Build the move plan using bulk SQL operations
plan, old_directories = await build_move_plan_bulk(sources, destination)
# Update status with total
self.update_status(
status='moving',
operation_total=len(plan),
)
# Update status with total
self.update_status(
status='moving',
operation_total=len(plan),
)
# Execute the plan in chunks
with flags.file_worker_discovery:
with get_db_session(commit=True) as session:
await self._execute_move_chunks(
plan, session, created_directories, revert_plan
)
# Execute the plan in chunks
with flags.file_worker_discovery:
with get_db_session(commit=True) as session:
await self._execute_move_chunks(
plan, session, created_directories, revert_plan
)
self._cleanup_old_directories(sources, old_directories)
await self._apply_post_processing()
self._cleanup_old_directories(sources, old_directories)
await self._apply_post_processing()
logger.info(f'Move completed: {len(plan)} items moved to {destination}')
Events.send_file_move_completed(f'Moved {len(sources)} items to {destination}')
@ -1800,6 +1913,499 @@ class FileWorker:
finally:
self.reset_status()
async def handle_reorganize(self, task: FileTask):
"""Handle reorganize task for collection file reorganization.
Unlike handle_move which moves all files to a single destination,
handle_reorganize handles arbitrary source->destination mappings
where each file (or FileGroup) can move to a different location.
This is used when a user changes their file_name_format configuration
and wants to reorganize existing files to match the new format.
Optimized for bulk operations:
- Single query to pre-fetch all FileGroups
- Uses FileGroup.files instead of filesystem globs
- Dict lookup for O(n) path matching
- Batched database updates
"""
from wrolpi.events import Events
from wrolpi.files.models import FileGroup
move_mappings = task.move_mappings
if not move_mappings:
logger.info('Reorganize task has no move mappings, nothing to do')
self._complete_job(task.job_id)
self.reset_status()
return
total_moves = len(move_mappings)
logger.info(f'Starting reorganize task with {total_moves} file moves')
# Transition job from 'pending' to 'running' so progress can be tracked
self._set_job_status(task.job_id, 'running')
self.update_status(
status='reorganizing',
task_type='reorganize',
paths=[],
destination=None,
operation_total=total_moves,
operation_processed=0,
operation_percent=0,
error=None,
collection_id=task.collection_id,
collection_kind=task.collection_kind,
)
# Track successful moves for rollback
completed_moves: list[tuple[pathlib.Path, pathlib.Path]] = []
# Track created directories for cleanup
created_directories: set[pathlib.Path] = set()
# Track source directories to clean up after moves
source_directories: set[pathlib.Path] = set()
try:
with flags.file_worker_discovery:
# Phase 1: Bulk pre-fetch all FileGroups
source_paths = [str(s) for s, d in move_mappings]
with get_db_session() as session:
file_groups = session.query(FileGroup).filter(
FileGroup.primary_path.in_(source_paths)
).all()
# Build lookup dict and detach from session
# Note: fg.primary_path is a pathlib.Path (from MediaPathType), but
# source_paths are strings, so we use str() for the dictionary keys
fg_by_path = {}
for fg in file_groups:
fg_by_path[str(fg.primary_path)] = {
'id': fg.id,
'directory': pathlib.Path(fg.directory),
'files': list(fg.files), # Copy the list
'data': dict(fg.data) if fg.data else None,
}
processed = 0
pending_updates = []
for source_path, dest_path in move_mappings:
source_path = pathlib.Path(source_path)
dest_path = pathlib.Path(dest_path)
source_exists = source_path.exists()
dest_exists = dest_path.exists()
# Look up the pre-fetched FileGroup
fg_info = fg_by_path.get(str(source_path))
if not fg_info:
logger.debug(f'FileGroup not found by primary_path: {source_path}')
processed += 1
continue
if source_exists and dest_exists:
# Conflict: both exist - skip with warning
logger.warning(f'Reorganize: both source and destination exist: {source_path} -> {dest_path}')
processed += 1
continue
if not source_exists and not dest_exists:
# Neither exists - skip with warning
logger.warning(f'Reorganize: neither source nor destination exist: {source_path}')
processed += 1
continue
if not source_exists and dest_exists:
# Recovery case: file already moved, just update DB to match destination
logger.info(f'Reorganize recovery: updating DB for already-moved file: {dest_path}')
dest_stem, _ = split_path_stem_and_suffix(dest_path)
# Find existing files at destination that match the FileGroup's file extensions
moved_files = []
for file_info in fg_info['files']:
_, suffix = split_path_stem_and_suffix(pathlib.Path(file_info['path']))
dest_file = dest_path.parent / f'{dest_stem}{suffix}'
if dest_file.exists():
# Map original source path to destination for DB update
src_file = fg_info['directory'] / file_info['path']
moved_files.append((src_file, dest_file))
if moved_files:
# Build DB update using existing destination files
moved_lookup = {src: dst for src, dst in moved_files}
new_files = []
for file_info in fg_info['files']:
old_file_path = pathlib.Path(file_info['path'])
if not old_file_path.is_absolute():
old_file_path = fg_info['directory'] / old_file_path
if old_file_path in moved_lookup:
file_info = dict(file_info)
file_info['path'] = moved_lookup[old_file_path].name
new_files.append(file_info)
new_data = fg_info['data']
if new_data:
new_data = dict(new_data)
filename_lookup = {src.name: dst.name for src, dst in moved_files}
for key, value in new_data.items():
if isinstance(value, str) and value in filename_lookup:
new_data[key] = filename_lookup[value]
elif isinstance(value, list):
new_data[key] = [filename_lookup.get(v, v) for v in value]
pending_updates.append({
'id': fg_info['id'],
'directory': str(dest_path.parent),
'primary_path': str(dest_path),
'files': new_files,
'data': new_data,
})
completed_moves.append((source_path, dest_path))
processed += 1
if processed % PROGRESS_UPDATE_INTERVAL == 0:
percent = int((processed / total_moves) * 100)
self.update_status(operation_processed=processed, operation_percent=percent)
continue
# Normal case: source exists, dest doesn't - move files
# Track source directory for cleanup
source_directories.add(source_path.parent)
# Create destination directory if needed
if not dest_path.parent.exists():
dest_path.parent.mkdir(parents=True, exist_ok=True)
created_directories.add(dest_path.parent)
# Phase 2: Use FileGroup.files instead of glob_shared_stem()
source_stem_files = []
for file_info in fg_info['files']:
file_path = pathlib.Path(file_info['path'])
if not file_path.is_absolute():
file_path = fg_info['directory'] / file_path
source_stem_files.append(file_path)
# Move all files in the FileGroup
dest_stem, _ = split_path_stem_and_suffix(dest_path)
moved_files = []
for src_file in source_stem_files:
if not src_file.exists():
continue
# Compute destination for this file based on its suffix
_, src_suffix = split_path_stem_and_suffix(src_file)
dest_file = dest_path.parent / f'{dest_stem}{src_suffix}'
try:
shutil.move(str(src_file), str(dest_file))
moved_files.append((src_file, dest_file))
logger.debug(f'Reorganize: moved {src_file} -> {dest_file}')
except Exception as e:
logger.error(f'Failed to move {src_file} -> {dest_file}: {e}')
raise
# Prepare FileGroup update
if moved_files:
# Phase 3: Use dict lookup instead of nested loop
moved_lookup = {src: dst for src, dst in moved_files}
# Update files list with new paths
new_files = []
for file_info in fg_info['files']:
old_file_path = pathlib.Path(file_info['path'])
if not old_file_path.is_absolute():
old_file_path = fg_info['directory'] / old_file_path
# Find the corresponding new path using dict lookup
if old_file_path in moved_lookup:
file_info = dict(file_info)
file_info['path'] = moved_lookup[old_file_path].name
new_files.append(file_info)
# Update data dict if it has path references
# Data fields store filenames only (no '/'), so use filename-to-filename lookup
new_data = fg_info['data']
if new_data:
new_data = dict(new_data)
# Build filename-to-filename lookup (not full paths)
filename_lookup = {src.name: dst.name for src, dst in moved_files}
for key, value in new_data.items():
if isinstance(value, str) and value in filename_lookup:
new_data[key] = filename_lookup[value]
elif isinstance(value, list):
# Handle list fields like caption_paths
new_data[key] = [filename_lookup.get(v, v) for v in value]
# Phase 4: Collect update for batch processing
pending_updates.append({
'id': fg_info['id'],
'directory': str(dest_path.parent),
'primary_path': str(dest_path),
'files': new_files,
'data': new_data,
})
completed_moves.append((source_path, dest_path))
# Flush every MOVE_CHUNK_SIZE items
if len(pending_updates) >= MOVE_CHUNK_SIZE:
_bulk_update_file_groups_reorganize(pending_updates)
pending_updates = []
processed += 1
# Update progress
if processed % PROGRESS_UPDATE_INTERVAL == 0 or processed == total_moves:
percent = int((processed / total_moves) * 100)
logger.info(f'Reorganize progress: {processed}/{total_moves} files ({percent}%)')
self.update_status(
operation_processed=processed,
operation_percent=percent,
)
# Final flush of pending updates
if pending_updates:
_bulk_update_file_groups_reorganize(pending_updates)
# Clean up empty source directories
for directory in sorted(source_directories, key=lambda p: len(p.parents), reverse=True):
try:
if directory.is_dir() and not any(directory.iterdir()):
delete_directory(directory)
logger.debug(f'Cleaned up empty directory: {directory}')
except (OSError, StopIteration):
pass
# Skip post-processing - reorganization only moves files, nothing to re-index
# Update file_format after successful completion (deferred update for resumability)
if task.collection_id and task.pending_file_format:
from wrolpi.collections.models import Collection
with get_db_session(commit=True) as session:
collection = session.query(Collection).filter_by(id=task.collection_id).one_or_none()
if collection:
collection.file_format = task.pending_file_format
logger.info(f'Updated file_format for collection {collection.name}')
# Trigger config save
if collection.kind == 'domain':
from modules.archive.lib import save_domains_config
save_domains_config.activate_switch()
elif collection.kind == 'channel':
from modules.videos.lib import save_channels_config
save_channels_config.activate_switch()
logger.info(f'Reorganize completed: {len(completed_moves)} files moved')
Events.send_file_move_completed(f'Reorganized {len(completed_moves)} files')
self._complete_job(task.job_id)
except asyncio.CancelledError:
logger.warning('Reorganize task was cancelled')
raise
except Exception as e:
logger.error(f'Reorganize failed: {e}', exc_info=e)
self.update_status(status='error', error=str(e))
# Note: We don't attempt automatic rollback for reorganize tasks
# as the partial state may be complex. User should refresh files.
Events.send_file_move_failed(f'Reorganize failed: {e}')
finally:
self.reset_status()
async def handle_batch_reorganize(self, task: FileTask):
"""Handle batch reorganization of multiple collections.
Processes collections sequentially:
1. For each collection, execute reorganization
2. Track overall progress and per-collection progress
3. Stop on first failure, report which collection failed
Progress tracking:
- overall_percent = (completed_collections + current_collection_%) / total_collections
- batch_status dict tracks completed list, current collection, and failure info
"""
from wrolpi.collections.reorganize import execute_reorganization
from wrolpi.collections.models import Collection
from wrolpi.events import Events
collection_ids = task.collection_ids or []
kind = task.batch_kind
if not collection_ids:
logger.info('Batch reorganize task has no collections, nothing to do')
self._complete_job(task.job_id)
self.reset_status()
return
total_collections = len(collection_ids)
logger.info(f'Starting batch reorganization of {total_collections} {kind} collections')
# Transition job from 'pending' to 'running'
self._set_job_status(task.job_id, 'running')
# Initialize batch status tracking
batch_status = {
'batch_job_id': task.job_id, # Include job ID so frontend can resume polling
'total_collections': total_collections,
'completed': [],
'current_collection': None,
'failed_collection': None,
'error': None,
'overall_percent': 0,
}
self.update_status(
status='batch_reorganizing',
task_type='batch_reorganize',
paths=[],
destination=None,
operation_total=total_collections,
operation_processed=0,
operation_percent=0,
error=None,
batch_status=batch_status,
collection_kind=kind,
)
try:
for idx, collection_id in enumerate(collection_ids):
# Fetch collection info
with get_db_session() as session:
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
if not collection:
logger.warning(f'Batch reorganize: collection {collection_id} not found, skipping')
continue
collection_name = collection.name
# Update current collection in status
batch_status['current_collection'] = {
'id': collection_id,
'name': collection_name,
'status': 'running',
'total': 0,
'completed': 0,
'percent': 0,
}
self.update_status(batch_status=batch_status)
logger.info(f'Batch reorganize: processing {collection_name} ({idx + 1}/{total_collections})')
try:
# Execute reorganization for this collection
# This queues a sub-job that we need to wait for
job_id = execute_reorganization(collection_id, session)
if job_id:
# Wait for the reorganization job to complete
# Poll for completion while updating batch status
await self._wait_for_sub_job_with_progress(
job_id, batch_status, collection_id, collection_name, idx, total_collections
)
# Mark this collection as completed
batch_status['completed'].append({
'id': collection_id,
'name': collection_name,
'status': 'complete',
})
# Update overall progress
completed_count = len(batch_status['completed'])
batch_status['overall_percent'] = int((completed_count / total_collections) * 100)
batch_status['current_collection'] = None
self.update_status(
operation_processed=completed_count,
operation_percent=batch_status['overall_percent'],
batch_status=batch_status,
)
logger.info(f'Batch reorganize: completed {collection_name}')
except Exception as e:
# Collection failed - stop batch processing
logger.error(f'Batch reorganize: {collection_name} failed: {e}')
batch_status['failed_collection'] = {
'id': collection_id,
'name': collection_name,
}
batch_status['error'] = str(e)
batch_status['current_collection'] = None
self.update_status(
status='error',
error=str(e),
batch_status=batch_status,
)
self._set_job_status(task.job_id, 'failed')
Events.send_file_move_failed(
f'Batch reorganization failed on {collection_name}: {e}'
)
return
# All collections completed successfully
logger.info(f'Batch reorganization completed: {len(batch_status["completed"])} collections processed')
Events.send_file_move_completed(
f'Batch reorganization completed: {len(batch_status["completed"])} {kind}s reorganized'
)
self._complete_job(task.job_id)
except asyncio.CancelledError:
logger.warning('Batch reorganize task was cancelled')
raise
except Exception as e:
logger.error(f'Batch reorganize failed: {e}', exc_info=e)
batch_status['error'] = str(e)
self.update_status(status='error', error=str(e), batch_status=batch_status)
self._set_job_status(task.job_id, 'failed')
Events.send_file_move_failed(f'Batch reorganization failed: {e}')
finally:
self.reset_status()
async def _wait_for_sub_job_with_progress(
self,
job_id: str,
batch_status: dict,
collection_id: int,
collection_name: str,
collection_idx: int,
total_collections: int,
):
"""Wait for a sub-job to complete while updating batch progress.
This is used by handle_batch_reorganize to wait for each collection's
reorganization job while keeping the batch status updated.
"""
import time
start = time.time()
timeout = 3600 # 1 hour max per collection
while time.time() - start < timeout:
status = self.get_job_status(job_id)
if status == 'complete':
return
# Transfer and process queue FIRST to ensure status is updated
self.transfer_queue()
await self.process_queue()
# Update current collection progress based on worker status (now fresh)
worker_status = self.status
current = batch_status.get('current_collection', {})
if current:
current['total'] = worker_status.get('operation_total', 0)
current['completed'] = worker_status.get('operation_processed', 0)
current['percent'] = worker_status.get('operation_percent', 0)
batch_status['current_collection'] = current
# Calculate overall percent including current progress
completed_count = len(batch_status.get('completed', []))
current_progress = current['percent'] / 100
batch_status['overall_percent'] = int(
((completed_count + current_progress) / total_collections) * 100
)
self.update_status(batch_status=batch_status)
await asyncio.sleep(0.05)
raise TimeoutError(f'Collection {collection_name} reorganization timed out')
# All processes create a FileWorker, but only one receives the signals and actually does work.
file_worker = FileWorker()

View file

@ -18,7 +18,11 @@ class MediaPathType(types.TypeDecorator): # noqa
raise ValueError('MediaPath cannot be empty')
if isinstance(value, pathlib.Path):
value = str(value.absolute())
if not value.is_absolute():
# Resolve relative paths against media directory, not cwd
from wrolpi.common import get_media_directory
value = get_media_directory() / value
value = str(value)
if not isinstance(value, str):
raise ValueError(f'Invalid MediaPath type ({type(value)}): {value}')

View file

@ -1015,29 +1015,60 @@ async def test_config_valid(async_client, test_wrolpi_config):
@pytest.mark.parametrize('name,expected_name', [
('foo', 'foo'),
(pathlib.Path('foo'), pathlib.Path('foo')),
# Short filenames with parents (no trimming needed) should preserve parent directories.
('some/parent/short.txt', 'some/parent/short.txt'),
('/absolute/parent/short.txt', '/absolute/parent/short.txt'),
(pathlib.Path('/some/parent/short.txt'), pathlib.Path('/some/parent/short.txt')),
# Long ASCII filenames are trimmed to 200 BYTES (not characters)
(
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.txt',
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt'
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt'
),
(
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json',
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
),
(
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json',
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
),
(
pathlib.Path(
'/some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.readability.json'),
pathlib.Path(
'/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json')
'/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json')
),
])
def test_trim_file_name(name, expected_name):
assert common.trim_file_name(name) == expected_name
def test_trim_file_name_utf8_bytes():
"""UTF-8 multi-byte characters must be trimmed by BYTE length, not character length.
Linux ext4/xfs filesystems have a 255 BYTE limit, not a 255 character limit.
Hindi/Devanagari characters are 3 bytes each in UTF-8, so a 121-character filename
can be 263 bytes, exceeding the limit.
This test reproduces the bug from: https://decay.local reorganization failure
"""
# This filename has 133 characters but 291 bytes - exceeds 255 byte limit
hindi_filename = '2022-04-13-19-40-51_भारतीय संस्कृति और परंपराएं बहुत समृद्ध हैं हमारे देश में विविधता में एकता है यह हमारी विरासत है.readability.html'
assert len(hindi_filename) == 133 # Characters
assert len(hindi_filename.encode('utf-8')) == 291 # Bytes - exceeds 255!
result = common.trim_file_name(hindi_filename)
# Result must fit within filesystem byte limit (200 bytes leaves room for suffix variations)
max_bytes = 200
result_bytes = len(result.encode('utf-8'))
assert result_bytes <= max_bytes, \
f'Trimmed filename is {result_bytes} bytes, exceeds {max_bytes} byte limit'
# Must preserve the suffix
assert result.endswith('.readability.html'), f'Suffix not preserved: {result}'
async def test_cached_multiprocessing_result(async_client):
call_count = 0

85
wrolpi/test_media_path.py Normal file
View file

@ -0,0 +1,85 @@
"""
Tests for MediaPathType to ensure paths are resolved correctly.
"""
import os
import pathlib
import pytest
from wrolpi.collections.models import Collection
from wrolpi.common import get_media_directory
from wrolpi.media_path import MediaPathType
def test_media_path_type_relative_path_resolved_against_media_directory(test_session, test_directory):
"""MediaPathType should resolve relative paths against media directory, not cwd."""
relative_path = pathlib.Path('archive/test.com')
collection = Collection(name='test.com', kind='domain', directory=relative_path)
test_session.add(collection)
test_session.commit()
test_session.expire(collection)
assert collection.directory.is_absolute()
assert collection.directory == test_directory / 'archive/test.com'
def test_media_path_type_process_bind_param_relative_path(test_directory):
"""process_bind_param should resolve relative paths against media directory."""
media_path_type = MediaPathType()
relative_path = pathlib.Path('archive/test.com')
result = media_path_type.process_bind_param(relative_path, dialect=None)
assert result == str(test_directory / 'archive/test.com')
assert not result.startswith(os.getcwd()) # NOT resolved against cwd
def test_collection_get_or_set_directory_database_roundtrip(test_session, test_directory, test_wrolpi_config):
"""get_or_set_directory should store and retrieve correct paths."""
from unittest import mock
collection = Collection(name='example.com', kind='domain')
test_session.add(collection)
test_session.commit()
# Mock switch activation since shared context isn't set up in unit tests
with mock.patch('modules.archive.lib.save_domains_config.activate_switch'):
result1 = collection.get_or_set_directory(test_session)
test_session.expire(collection)
result2 = collection.get_or_set_directory(test_session)
assert result1 == result2
assert str(test_directory) in str(result2)
assert '/opt/wrolpi' not in str(collection.directory)
def test_media_path_type_absolute_path_unchanged(test_session, test_directory):
"""Absolute paths under media directory should be stored unchanged."""
absolute_path = test_directory / 'archive/test.com'
collection = Collection(name='test.com', kind='domain', directory=absolute_path)
test_session.add(collection)
test_session.commit()
test_session.expire(collection)
assert collection.directory == absolute_path
def test_media_path_type_none_unchanged(test_directory):
"""None should pass through unchanged."""
media_path_type = MediaPathType()
result = media_path_type.process_bind_param(None, dialect=None)
assert result is None
def test_media_path_type_empty_string_raises(test_directory):
"""Empty string should raise ValueError."""
media_path_type = MediaPathType()
with pytest.raises(ValueError, match='MediaPath cannot be empty'):
media_path_type.process_bind_param('', dialect=None)
def test_media_path_type_string_path_passed_through(test_directory):
"""String paths should be passed through as-is (assumed already absolute)."""
media_path_type = MediaPathType()
path_str = str(test_directory / 'archive/test.com')
result = media_path_type.process_bind_param(path_str, dialect=None)
assert result == path_str