mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
FileGroups in Collections can be reorganized in the Collection's directory.
* Batch reorganization is supported for both Domain and Channel Collections. * Batch reorganization happens when any Collection does not match the respective file format. For example, a Channel Collection file format that does not match the `yt_dlp_options.file_name_format` will show `needs_reorganization`. * Individual Collections can always be reorganized. The reorganization is a little more aggressive. Useful for when some files may have been missed in the batch reorganization.
This commit is contained in:
parent
208ac4fac7
commit
6891ebeb62
49 changed files with 6614 additions and 181 deletions
48
alembic/versions/a1b2c3d4e5f6_add_collection_file_format.py
Normal file
48
alembic/versions/a1b2c3d4e5f6_add_collection_file_format.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
"""add_collection_file_format
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises: 76add63ed225
|
||||
Create Date: 2026-01-23
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'a1b2c3d4e5f6'
|
||||
down_revision = '76add63ed225'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# Add file_format column to collection table
|
||||
# This stores the file_name_format used when files were last organized
|
||||
op.add_column('collection', sa.Column('file_format', sa.String(), nullable=True))
|
||||
|
||||
# Set default values from current configs so existing collections don't need reorganization
|
||||
from modules.archive.lib import get_archive_downloader_config
|
||||
from modules.videos.lib import get_videos_downloader_config
|
||||
|
||||
connection = op.get_bind()
|
||||
|
||||
# Update domain collections with archive config's file_name_format
|
||||
archive_config = get_archive_downloader_config()
|
||||
if archive_config.file_name_format:
|
||||
connection.execute(
|
||||
sa.text("UPDATE collection SET file_format = :format WHERE kind = 'domain'"),
|
||||
{'format': archive_config.file_name_format}
|
||||
)
|
||||
|
||||
# Update channel collections with video config's file_name_format
|
||||
video_config = get_videos_downloader_config()
|
||||
if video_config.file_name_format:
|
||||
connection.execute(
|
||||
sa.text("UPDATE collection SET file_format = :format WHERE kind = 'channel'"),
|
||||
{'format': video_config.file_name_format}
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column('collection', 'file_format')
|
||||
189
alembic/versions/b2b9700f5048_fix_corrupted_media_paths.py
Normal file
189
alembic/versions/b2b9700f5048_fix_corrupted_media_paths.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
"""Fix media paths incorrectly resolved against /opt/wrolpi.
|
||||
|
||||
MediaPathType.process_bind_param() was calling .absolute() on relative paths,
|
||||
which resolves against the current working directory (cwd) instead of the
|
||||
media directory. In production, cwd is /opt/wrolpi, so paths like
|
||||
'archive/spiegel.de' became '/opt/wrolpi/archive/spiegel.de' instead of
|
||||
'/media/wrolpi/archive/spiegel.de'.
|
||||
|
||||
This migration fixes any corrupted paths in the database and saves the
|
||||
corrected paths to config files.
|
||||
|
||||
Revision ID: b2b9700f5048
|
||||
Revises: a1b2c3d4e5f6
|
||||
Create Date: 2026-01-27
|
||||
"""
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
import yaml
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'b2b9700f5048'
|
||||
down_revision = 'a1b2c3d4e5f6'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def get_relative_path(absolute_path: str, media_directory: str) -> str:
|
||||
"""Convert absolute path to relative path under media directory."""
|
||||
if absolute_path and absolute_path.startswith(media_directory):
|
||||
return absolute_path[len(media_directory):].lstrip('/')
|
||||
return absolute_path
|
||||
|
||||
|
||||
def save_config_file(config: dict, file_path: pathlib.Path, width: int = 120):
|
||||
"""Write config dict to YAML file."""
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with file_path.open('wt') as fh:
|
||||
yaml.dump(config, fh, width=width, sort_keys=True)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
|
||||
|
||||
def upgrade():
|
||||
connection = op.get_bind()
|
||||
|
||||
# Fix collection.directory
|
||||
result = connection.execute(sa.text("""
|
||||
UPDATE collection
|
||||
SET directory = REPLACE(directory, '/opt/wrolpi/', '/media/wrolpi/')
|
||||
WHERE directory LIKE '/opt/wrolpi/%'
|
||||
"""))
|
||||
if result.rowcount:
|
||||
print(f"Fixed {result.rowcount} collection directory paths")
|
||||
|
||||
# Fix file_group.directory
|
||||
result = connection.execute(sa.text("""
|
||||
UPDATE file_group
|
||||
SET directory = REPLACE(directory, '/opt/wrolpi/', '/media/wrolpi/')
|
||||
WHERE directory LIKE '/opt/wrolpi/%'
|
||||
"""))
|
||||
if result.rowcount:
|
||||
print(f"Fixed {result.rowcount} file_group directory paths")
|
||||
|
||||
# Fix file_group.primary_path
|
||||
result = connection.execute(sa.text("""
|
||||
UPDATE file_group
|
||||
SET primary_path = REPLACE(primary_path, '/opt/wrolpi/', '/media/wrolpi/')
|
||||
WHERE primary_path LIKE '/opt/wrolpi/%'
|
||||
"""))
|
||||
if result.rowcount:
|
||||
print(f"Fixed {result.rowcount} file_group primary_path paths")
|
||||
|
||||
# Fix download.destination
|
||||
result = connection.execute(sa.text("""
|
||||
UPDATE download
|
||||
SET destination = REPLACE(destination, '/opt/wrolpi/', '/media/wrolpi/')
|
||||
WHERE destination LIKE '/opt/wrolpi/%'
|
||||
"""))
|
||||
if result.rowcount:
|
||||
print(f"Fixed {result.rowcount} download destination paths")
|
||||
|
||||
# Save corrected paths to config files directly (bypass config framework which needs SharedContext)
|
||||
try:
|
||||
from wrolpi.common import get_media_directory
|
||||
media_dir = str(get_media_directory())
|
||||
config_dir = get_media_directory() / 'config'
|
||||
|
||||
# Save domains config
|
||||
domains_result = connection.execute(sa.text("""
|
||||
SELECT id, name, kind, description, directory, file_format
|
||||
FROM collection
|
||||
WHERE kind = 'domain'
|
||||
ORDER BY name
|
||||
"""))
|
||||
domains_collections = []
|
||||
for row in domains_result:
|
||||
collection_config = {'name': row[1], 'kind': row[2]}
|
||||
if row[3]: # description
|
||||
collection_config['description'] = row[3]
|
||||
if row[4]: # directory
|
||||
collection_config['directory'] = get_relative_path(row[4], media_dir + '/')
|
||||
if row[5]: # file_format
|
||||
collection_config['file_format'] = row[5]
|
||||
# Get tag name if exists
|
||||
tag_result = connection.execute(sa.text("""
|
||||
SELECT t.name FROM tag t
|
||||
JOIN collection c ON c.tag_id = t.id
|
||||
WHERE c.id = :id
|
||||
"""), {'id': row[0]}).first()
|
||||
collection_config['tag_name'] = tag_result[0] if tag_result else None
|
||||
domains_collections.append(collection_config)
|
||||
|
||||
# Read existing config to preserve version
|
||||
domains_file = config_dir / 'domains.yaml'
|
||||
version = 0
|
||||
if domains_file.exists():
|
||||
with domains_file.open() as f:
|
||||
existing = yaml.safe_load(f) or {}
|
||||
version = existing.get('version', 0)
|
||||
|
||||
domains_config = {'version': version + 1, 'collections': domains_collections}
|
||||
save_config_file(domains_config, domains_file)
|
||||
print(f"Saved domains config with {len(domains_collections)} collections")
|
||||
|
||||
# Save channels config
|
||||
# Columns: 0=id, 1=name, 2=kind, 3=description, 4=directory, 5=file_format,
|
||||
# 6=url, 7=source_id, 8=generate_posters, 9=calculate_duration, 10=download_missing_data
|
||||
channels_result = connection.execute(sa.text("""
|
||||
SELECT c.id, c.name, c.kind, c.description, c.directory, c.file_format,
|
||||
ch.url, ch.source_id, ch.generate_posters, ch.calculate_duration,
|
||||
ch.download_missing_data
|
||||
FROM collection c
|
||||
JOIN channel ch ON ch.collection_id = c.id
|
||||
WHERE c.kind = 'channel'
|
||||
ORDER BY c.name
|
||||
"""))
|
||||
channels_list = []
|
||||
for row in channels_result:
|
||||
channel_config = {'name': row[1]}
|
||||
if row[4]: # directory
|
||||
channel_config['directory'] = get_relative_path(row[4], media_dir + '/')
|
||||
if row[6]: # url
|
||||
channel_config['url'] = row[6]
|
||||
if row[7]: # source_id
|
||||
channel_config['source_id'] = row[7]
|
||||
channel_config['generate_posters'] = row[8]
|
||||
channel_config['calculate_duration'] = row[9]
|
||||
channel_config['download_missing_data'] = row[10]
|
||||
if row[5]: # file_format
|
||||
channel_config['file_format'] = row[5]
|
||||
# Get tag name if exists
|
||||
tag_result = connection.execute(sa.text("""
|
||||
SELECT t.name FROM tag t
|
||||
JOIN collection c ON c.tag_id = t.id
|
||||
WHERE c.id = :id
|
||||
"""), {'id': row[0]}).first()
|
||||
channel_config['tag_name'] = tag_result[0] if tag_result else None
|
||||
# Get downloads for this channel's collection
|
||||
downloads_result = connection.execute(sa.text("""
|
||||
SELECT url, frequency FROM download
|
||||
WHERE collection_id = :collection_id
|
||||
"""), {'collection_id': row[0]})
|
||||
downloads = [{'url': d[0], 'frequency': d[1]} for d in downloads_result]
|
||||
if downloads:
|
||||
channel_config['downloads'] = downloads
|
||||
channels_list.append(channel_config)
|
||||
|
||||
# Read existing config to preserve version
|
||||
channels_file = config_dir / 'channels.yaml'
|
||||
version = 0
|
||||
if channels_file.exists():
|
||||
with channels_file.open() as f:
|
||||
existing = yaml.safe_load(f) or {}
|
||||
version = existing.get('version', 0)
|
||||
|
||||
channels_config = {'version': version + 1, 'channels': channels_list}
|
||||
save_config_file(channels_config, channels_file)
|
||||
print(f"Saved channels config with {len(channels_list)} channels")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not save configs: {e}")
|
||||
|
||||
|
||||
def downgrade():
|
||||
# Data fix, no meaningful downgrade
|
||||
pass
|
||||
|
|
@ -11,6 +11,7 @@ import {InventoryRoute} from "./components/Inventory";
|
|||
import {ArchiveRoute} from "./components/Archive";
|
||||
import {FilesRoute} from "./components/Files";
|
||||
import {QueryProvider, StatusProvider} from "./hooks/customHooks";
|
||||
import {FileWorkerStatusProvider} from "./contexts/FileWorkerStatusContext";
|
||||
import {MapRoute} from "./components/Map";
|
||||
import {MediaContextProvider, mediaStyles, StatusContext, ThemeContext} from "./contexts/contexts";
|
||||
import {Header, ThemeProvider} from "./components/Theme";
|
||||
|
|
@ -98,12 +99,14 @@ export default function App() {
|
|||
useEventsInterval();
|
||||
|
||||
return <StatusProvider>
|
||||
{/* Context and style to handle switching between mobile/computer. */}
|
||||
<style>{mediaStyles}</style>
|
||||
{/* Toasts can be on any page. */}
|
||||
<SemanticToastContainer position='top-right'/>
|
||||
<MediaContextProvider>
|
||||
<RouterProvider router={router}/>
|
||||
</MediaContextProvider>
|
||||
<FileWorkerStatusProvider>
|
||||
{/* Context and style to handle switching between mobile/computer. */}
|
||||
<style>{mediaStyles}</style>
|
||||
{/* Toasts can be on any page. */}
|
||||
<SemanticToastContainer position='top-right'/>
|
||||
<MediaContextProvider>
|
||||
<RouterProvider router={router}/>
|
||||
</MediaContextProvider>
|
||||
</FileWorkerStatusProvider>
|
||||
</StatusProvider>
|
||||
}
|
||||
|
|
|
|||
128
app/src/api.js
128
app/src/api.js
|
|
@ -442,6 +442,14 @@ export async function postArchiveFileFormat(archive_file_format) {
|
|||
return await apiPost(`${ARCHIVES_API}/file_format`, body);
|
||||
}
|
||||
|
||||
export async function fetchArchiveBrowsers() {
|
||||
const response = await apiGet(`${ARCHIVES_API}/browsers`);
|
||||
if (response.ok) {
|
||||
return await response.json();
|
||||
}
|
||||
return {browsers: [], available: false};
|
||||
}
|
||||
|
||||
export async function fetchBrowserProfiles() {
|
||||
const response = await apiGet(`${API_URI}/videos/browser-profiles`);
|
||||
if (response.status === 200) {
|
||||
|
|
@ -888,6 +896,126 @@ export async function refreshDomain(domainId) {
|
|||
return response;
|
||||
}
|
||||
|
||||
export async function previewCollectionReorganization(collectionId) {
|
||||
const response = await apiGet(`${COLLECTIONS_API}/${collectionId}/reorganize/preview`);
|
||||
if (response.ok) {
|
||||
return await response.json();
|
||||
} else {
|
||||
const message = await getErrorMessage(response, 'Unable to preview reorganization. See server logs.');
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Reorganization Preview Error',
|
||||
description: message,
|
||||
time: 5000,
|
||||
});
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
export async function executeCollectionReorganization(collectionId) {
|
||||
const response = await apiPost(`${COLLECTIONS_API}/${collectionId}/reorganize`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
if (data.job_id) {
|
||||
toast({
|
||||
type: 'success',
|
||||
title: 'Reorganization Started',
|
||||
description: 'Files are being reorganized in the background',
|
||||
time: 3000,
|
||||
});
|
||||
} else {
|
||||
toast({
|
||||
type: 'info',
|
||||
title: 'No Changes Needed',
|
||||
description: data.message || 'No files need reorganization',
|
||||
time: 3000,
|
||||
});
|
||||
}
|
||||
return data;
|
||||
} else {
|
||||
const message = await getErrorMessage(response, 'Unable to start reorganization. See server logs.');
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Reorganization Error',
|
||||
description: message,
|
||||
time: 5000,
|
||||
});
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getReorganizationStatus(collectionId, jobId) {
|
||||
const response = await apiGet(`${COLLECTIONS_API}/${collectionId}/reorganize/status/${jobId}`);
|
||||
if (response.ok) {
|
||||
return await response.json();
|
||||
} else {
|
||||
const message = await getErrorMessage(response, 'Unable to get reorganization status.');
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
// Batch Reorganization API Functions
|
||||
|
||||
export async function previewBatchReorganization(kind) {
|
||||
const endpoint = kind === 'channel' ? 'channels' : 'domains';
|
||||
const response = await apiGet(`${COLLECTIONS_API}/reorganize/${endpoint}`);
|
||||
if (response.ok) {
|
||||
return await response.json();
|
||||
} else {
|
||||
const message = await getErrorMessage(response, `Unable to get ${kind}s needing reorganization.`);
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Batch Reorganization Preview Error',
|
||||
description: message,
|
||||
time: 5000,
|
||||
});
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
export async function executeBatchReorganization(kind) {
|
||||
const endpoint = kind === 'channel' ? 'channels' : 'domains';
|
||||
const response = await apiPost(`${COLLECTIONS_API}/reorganize/${endpoint}`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
if (data.batch_job_id) {
|
||||
toast({
|
||||
type: 'success',
|
||||
title: 'Batch Reorganization Started',
|
||||
description: `${data.collection_count} ${kind}s are being reorganized`,
|
||||
time: 3000,
|
||||
});
|
||||
} else {
|
||||
toast({
|
||||
type: 'info',
|
||||
title: 'No Changes Needed',
|
||||
description: data.message || `No ${kind}s need reorganization`,
|
||||
time: 3000,
|
||||
});
|
||||
}
|
||||
return data;
|
||||
} else {
|
||||
const message = await getErrorMessage(response, `Unable to start batch reorganization. See server logs.`);
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Batch Reorganization Error',
|
||||
description: message,
|
||||
time: 5000,
|
||||
});
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getBatchReorganizationStatus(batchJobId) {
|
||||
const response = await apiGet(`${COLLECTIONS_API}/reorganize/batch/status/${batchJobId}`);
|
||||
if (response.ok) {
|
||||
return await response.json();
|
||||
} else {
|
||||
const message = await getErrorMessage(response, 'Unable to get batch reorganization status.');
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getArchive(archiveId) {
|
||||
const response = await apiGet(`${ARCHIVES_API}/${archiveId}`);
|
||||
if (response.ok) {
|
||||
|
|
|
|||
|
|
@ -44,11 +44,13 @@ import {
|
|||
import {
|
||||
deleteArchives,
|
||||
deleteDomain,
|
||||
fetchArchiveBrowsers,
|
||||
fetchArchiveDownloaderConfig,
|
||||
generateArchiveScreenshot,
|
||||
getCollectionTagInfo,
|
||||
postArchiveFileFormat,
|
||||
postDownload,
|
||||
previewBatchReorganization,
|
||||
refreshDomain,
|
||||
tagDomain,
|
||||
tagFileGroup,
|
||||
|
|
@ -57,10 +59,13 @@ import {
|
|||
} from "../api";
|
||||
import {InputForm, useForm} from "../hooks/useForm";
|
||||
import {CollectionTagModal} from "./collections/CollectionTagModal";
|
||||
import {CollectionReorganizeModal} from "./collections/CollectionReorganizeModal";
|
||||
import {BatchReorganizeModal} from "./collections/BatchReorganizeModal";
|
||||
import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {
|
||||
useArchive,
|
||||
useDockerized,
|
||||
useDomain,
|
||||
useDomains,
|
||||
useOneQuery,
|
||||
|
|
@ -505,6 +510,8 @@ export function DomainEditPage() {
|
|||
|
||||
// Modal state for tagging
|
||||
const [tagEditModalOpen, setTagEditModalOpen] = useState(false);
|
||||
// Modal state for reorganization
|
||||
const [reorganizeModalOpen, setReorganizeModalOpen] = useState(false);
|
||||
|
||||
useTitle(`Edit Domain: ${domain?.domain || '...'}`);
|
||||
|
||||
|
|
@ -612,10 +619,21 @@ export function DomainEditPage() {
|
|||
style={{marginTop: '1em'}}
|
||||
>Tag</Button>;
|
||||
|
||||
const reorganizeButton = (
|
||||
<APIButton
|
||||
color='orange'
|
||||
size='small'
|
||||
onClick={() => setReorganizeModalOpen(true)}
|
||||
obeyWROLMode={true}
|
||||
style={{marginTop: '1em'}}
|
||||
>Reorganize Files</APIButton>
|
||||
);
|
||||
|
||||
const actionButtons = <>
|
||||
{deleteButton}
|
||||
{refreshButton}
|
||||
{tagButton}
|
||||
{reorganizeButton}
|
||||
</>;
|
||||
|
||||
const [descriptionProps] = form.getCustomProps({name: 'description', path: 'description'});
|
||||
|
|
@ -626,6 +644,16 @@ export function DomainEditPage() {
|
|||
<Button>Archives</Button>
|
||||
</Link>
|
||||
|
||||
{domain?.needs_reorganization && (
|
||||
<Message warning>
|
||||
<Message.Header>File Format Changed</Message.Header>
|
||||
<p>
|
||||
The file name format has changed. Click "Reorganize Files" to move existing files
|
||||
to match the new format.
|
||||
</p>
|
||||
</Message>
|
||||
)}
|
||||
|
||||
<CollectionEditForm
|
||||
form={form}
|
||||
title={`Edit Domain: ${domain?.domain || '...'}`}
|
||||
|
|
@ -670,6 +698,16 @@ export function DomainEditPage() {
|
|||
hasDirectory={!!domain?.directory}
|
||||
/>
|
||||
|
||||
{/* Reorganize Modal */}
|
||||
<CollectionReorganizeModal
|
||||
open={reorganizeModalOpen}
|
||||
onClose={() => setReorganizeModalOpen(false)}
|
||||
collectionId={domain?.id}
|
||||
collectionName={domain?.domain}
|
||||
onComplete={fetchDomain}
|
||||
needsReorganization={domain?.needs_reorganization}
|
||||
/>
|
||||
|
||||
{/* Downloads Segment */}
|
||||
<Segment>
|
||||
<Header as='h1'>Downloads</Header>
|
||||
|
|
@ -726,13 +764,145 @@ function ArchiveFileNameForm({form}) {
|
|||
/>
|
||||
}
|
||||
|
||||
function BrowserConfigForm({form, browsers, browsersAvailable}) {
|
||||
const {t} = React.useContext(ThemeContext);
|
||||
const [useCustomPath, setUseCustomPath] = useState(false);
|
||||
|
||||
// Determine if custom path is being used based on current value
|
||||
React.useEffect(() => {
|
||||
const currentBrowser = form.formData?.browser_executable;
|
||||
if (currentBrowser) {
|
||||
// Check if current value matches any known browser
|
||||
const isKnownBrowser = browsers.some(b => b.path === currentBrowser || b.key === currentBrowser);
|
||||
setUseCustomPath(!isKnownBrowser);
|
||||
}
|
||||
}, [form.formData?.browser_executable, browsers]);
|
||||
|
||||
if (!browsersAvailable) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Build dropdown options: auto-detect + installed browsers + custom
|
||||
const browserOptions = [
|
||||
{key: 'auto', value: '', text: 'Auto-detect (recommended)'},
|
||||
...browsers.map(b => ({key: b.key, value: b.path, text: `${b.name} (${b.path})`})),
|
||||
{key: 'custom', value: '__custom__', text: 'Custom path...'},
|
||||
];
|
||||
|
||||
const handleBrowserChange = (e, {value}) => {
|
||||
if (value === '__custom__') {
|
||||
setUseCustomPath(true);
|
||||
form.setValue('browser_executable', '');
|
||||
} else {
|
||||
setUseCustomPath(false);
|
||||
form.setValue('browser_executable', value || null);
|
||||
}
|
||||
};
|
||||
|
||||
const currentValue = form.formData?.browser_executable || '';
|
||||
const dropdownValue = useCustomPath ? '__custom__' : currentValue;
|
||||
|
||||
return <>
|
||||
<Header as='h4'>Browser Settings</Header>
|
||||
<p {...t}>
|
||||
Configure which browser SingleFile uses to create archives.
|
||||
These settings only apply to native deployments (Raspberry Pi/Debian).
|
||||
</p>
|
||||
|
||||
<Form.Field>
|
||||
<label {...t}>Browser</label>
|
||||
<Form.Dropdown
|
||||
selection
|
||||
options={browserOptions}
|
||||
value={dropdownValue}
|
||||
onChange={handleBrowserChange}
|
||||
placeholder='Select browser...'
|
||||
/>
|
||||
</Form.Field>
|
||||
|
||||
{useCustomPath && (
|
||||
<Form.Field>
|
||||
<label {...t}>Custom Browser Path</label>
|
||||
<Input
|
||||
fluid
|
||||
placeholder='/usr/bin/chromium'
|
||||
value={form.formData?.browser_executable || ''}
|
||||
onChange={(e) => form.setValue('browser_executable', e.target.value)}
|
||||
/>
|
||||
<small {...t}>Enter the absolute path to the browser executable.</small>
|
||||
</Form.Field>
|
||||
)}
|
||||
|
||||
<Form.Field>
|
||||
<label {...t}>Browser Arguments</label>
|
||||
<Input
|
||||
fluid
|
||||
placeholder='["--no-sandbox"]'
|
||||
value={form.formData?.browser_args || '["--no-sandbox"]'}
|
||||
onChange={(e) => form.setValue('browser_args', e.target.value)}
|
||||
/>
|
||||
<small {...t}>JSON array of arguments passed to the browser. Default: ["--no-sandbox"]</small>
|
||||
</Form.Field>
|
||||
|
||||
<Form.Field>
|
||||
<label {...t}>User Agent</label>
|
||||
<Input
|
||||
fluid
|
||||
placeholder='Leave empty to use system default'
|
||||
value={form.formData?.user_agent || ''}
|
||||
onChange={(e) => form.setValue('user_agent', e.target.value || null)}
|
||||
/>
|
||||
<small {...t}>Custom user agent string. Leave empty to use the system default.</small>
|
||||
</Form.Field>
|
||||
</>;
|
||||
}
|
||||
|
||||
function ArchiveSettingsPage() {
|
||||
useTitle('Archive Settings');
|
||||
|
||||
const {t} = React.useContext(ThemeContext);
|
||||
const dockerized = useDockerized();
|
||||
const [batchModalOpen, setBatchModalOpen] = useState(false);
|
||||
const [domainsNeedingReorg, setDomainsNeedingReorg] = useState(0);
|
||||
const [fetchingReorgCount, setFetchingReorgCount] = useState(true);
|
||||
const [browsers, setBrowsers] = useState([]);
|
||||
const [browsersAvailable, setBrowsersAvailable] = useState(false);
|
||||
|
||||
// Check how many domains need reorganization on mount
|
||||
React.useEffect(() => {
|
||||
setFetchingReorgCount(true);
|
||||
previewBatchReorganization('domain')
|
||||
.then(data => {
|
||||
setDomainsNeedingReorg(data.total_collections || 0);
|
||||
})
|
||||
.catch(() => {
|
||||
setDomainsNeedingReorg(0);
|
||||
})
|
||||
.finally(() => {
|
||||
setFetchingReorgCount(false);
|
||||
});
|
||||
}, []);
|
||||
|
||||
// Fetch available browsers on mount (only if not dockerized)
|
||||
React.useEffect(() => {
|
||||
if (dockerized === false) {
|
||||
fetchArchiveBrowsers()
|
||||
.then(data => {
|
||||
setBrowsers(data.browsers || []);
|
||||
setBrowsersAvailable(data.available === true);
|
||||
})
|
||||
.catch(() => {
|
||||
setBrowsers([]);
|
||||
setBrowsersAvailable(false);
|
||||
});
|
||||
}
|
||||
}, [dockerized]);
|
||||
|
||||
const emptyFormData = {
|
||||
file_name_format: '%(download_datetime)s_%(title)s.%(ext)s',
|
||||
browser_executable: null,
|
||||
browser_args: '["--no-sandbox"]',
|
||||
user_agent: null,
|
||||
};
|
||||
|
||||
const configSubmitter = async () => {
|
||||
|
|
@ -769,6 +939,20 @@ function ArchiveSettingsPage() {
|
|||
<ArchiveFileNameForm form={configForm}/>
|
||||
</GridColumn>
|
||||
</GridRow>
|
||||
|
||||
{/* Browser settings - only shown on native deployments */}
|
||||
{!dockerized && browsersAvailable && (
|
||||
<GridRow columns={1}>
|
||||
<GridColumn mobile={16} computer={8}>
|
||||
<BrowserConfigForm
|
||||
form={configForm}
|
||||
browsers={browsers}
|
||||
browsersAvailable={browsersAvailable}
|
||||
/>
|
||||
</GridColumn>
|
||||
</GridRow>
|
||||
)}
|
||||
|
||||
<GridRow columns={1}>
|
||||
<GridColumn textAlign='right'>
|
||||
<APIButton
|
||||
|
|
@ -784,6 +968,40 @@ function ArchiveSettingsPage() {
|
|||
</Form>
|
||||
</Segment>
|
||||
|
||||
<Segment>
|
||||
<Header as='h4'>File Organization</Header>
|
||||
<p>
|
||||
{fetchingReorgCount
|
||||
? 'Checking for domains that need reorganization...'
|
||||
: <>
|
||||
<strong>{domainsNeedingReorg}</strong> domain{domainsNeedingReorg !== 1 ? 's' : ''}
|
||||
{domainsNeedingReorg > 0
|
||||
? ' have files that do not match the current file name format.'
|
||||
: '. All domains are organized correctly.'}
|
||||
</>
|
||||
}
|
||||
</p>
|
||||
<Button
|
||||
color='orange'
|
||||
onClick={() => setBatchModalOpen(true)}
|
||||
id='reorganize_all_domains_button'
|
||||
disabled={fetchingReorgCount || domainsNeedingReorg === 0}
|
||||
loading={fetchingReorgCount}
|
||||
>
|
||||
<Icon name='folder open outline'/> Reorganize All Domains
|
||||
</Button>
|
||||
</Segment>
|
||||
|
||||
<BatchReorganizeModal
|
||||
open={batchModalOpen}
|
||||
onClose={() => setBatchModalOpen(false)}
|
||||
kind='domain'
|
||||
onComplete={() => {
|
||||
setBatchModalOpen(false);
|
||||
setDomainsNeedingReorg(0);
|
||||
}}
|
||||
/>
|
||||
|
||||
<Segment>
|
||||
<Header as='h3'>SingleFile Browser Extension</Header>
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import React, {useState} from "react";
|
|||
import {Grid, StatisticLabel, StatisticValue,} from "semantic-ui-react";
|
||||
import {createChannel, deleteChannel, refreshChannel, tagChannel, tagChannelInfo} from "../api";
|
||||
import {CollectionTagModal} from "./collections/CollectionTagModal";
|
||||
import {CollectionReorganizeModal} from "./collections/CollectionReorganizeModal";
|
||||
import {
|
||||
APIButton,
|
||||
BackButton,
|
||||
|
|
@ -88,6 +89,7 @@ export function ChannelEditPage() {
|
|||
|
||||
const [tagEditModalOpen, setTagEditModalOpen] = useState(false);
|
||||
const [downloadModalOpen, setDownloadModalOpen] = useState(false);
|
||||
const [reorganizeModalOpen, setReorganizeModalOpen] = useState(false);
|
||||
|
||||
const {channel, form, fetchChannel} = useChannel(channelId);
|
||||
|
||||
|
|
@ -192,10 +194,21 @@ export function ChannelEditPage() {
|
|||
style={{marginTop: '1em'}}
|
||||
>Tag</Button>;
|
||||
|
||||
const reorganizeButton = (
|
||||
<APIButton
|
||||
color='orange'
|
||||
size='small'
|
||||
onClick={() => setReorganizeModalOpen(true)}
|
||||
obeyWROLMode={true}
|
||||
style={{marginTop: '1em'}}
|
||||
>Reorganize Files</APIButton>
|
||||
);
|
||||
|
||||
const actionButtons = <>
|
||||
{deleteButton}
|
||||
{refreshButton}
|
||||
{tagButton}
|
||||
{reorganizeButton}
|
||||
</>;
|
||||
|
||||
const downloadMissingDataInfo = 'Automatically download missing comments, etc, in the background.';
|
||||
|
|
@ -207,6 +220,16 @@ export function ChannelEditPage() {
|
|||
<Button>Videos</Button>
|
||||
</Link>
|
||||
|
||||
{channel?.needs_reorganization && (
|
||||
<Message warning>
|
||||
<Message.Header>File Format Changed</Message.Header>
|
||||
<p>
|
||||
The file name format has changed. Click "Reorganize Files" to move existing files
|
||||
to match the new format.
|
||||
</p>
|
||||
</Message>
|
||||
)}
|
||||
|
||||
<CollectionEditForm
|
||||
form={form}
|
||||
title="Edit Channel"
|
||||
|
|
@ -279,6 +302,16 @@ export function ChannelEditPage() {
|
|||
hasDirectory={!!channel.directory}
|
||||
/>
|
||||
|
||||
{/* Reorganize Modal */}
|
||||
<CollectionReorganizeModal
|
||||
open={reorganizeModalOpen}
|
||||
onClose={() => setReorganizeModalOpen(false)}
|
||||
collectionId={channel?.collection_id}
|
||||
collectionName={channel?.name}
|
||||
onComplete={fetchChannel}
|
||||
needsReorganization={channel?.needs_reorganization}
|
||||
/>
|
||||
|
||||
{/* Downloads Segment */}
|
||||
<Segment>
|
||||
<Grid columns={2}>
|
||||
|
|
|
|||
|
|
@ -780,7 +780,7 @@ export function InfoPopup({
|
|||
position = 'left center',
|
||||
iconSize = null,
|
||||
header = '',
|
||||
iconStyle = {marginLeft: '0.25em', marginRight: '0.25em', marginTop: '-0.5em'},
|
||||
iconStyle = { margin: '0.5em'},
|
||||
...props
|
||||
}) {
|
||||
const trigger = <Icon link name={icon} size={iconSize} style={iconStyle}/>;
|
||||
|
|
|
|||
|
|
@ -54,6 +54,19 @@ jest.mock('./collections/CollectionTagModal', () => ({
|
|||
},
|
||||
}));
|
||||
|
||||
// Mock CollectionReorganizeModal
|
||||
jest.mock('./collections/CollectionReorganizeModal', () => ({
|
||||
CollectionReorganizeModal: ({open, onClose, collectionId, collectionName, onComplete}) => {
|
||||
if (!open) return null;
|
||||
return (
|
||||
<div data-testid="collection-reorganize-modal">
|
||||
<div data-testid="modal-header">Reorganize Files: {collectionName}</div>
|
||||
<button data-testid="close-button" onClick={onClose}>Close</button>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
}));
|
||||
|
||||
// Mock API functions
|
||||
const mockGetCollectionTagInfo = jest.fn();
|
||||
jest.mock('../api', () => ({
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ import {
|
|||
useTitle
|
||||
} from "./Common";
|
||||
import {
|
||||
useFilesProgressInterval,
|
||||
usePages,
|
||||
useSearchFiles,
|
||||
useSearchFilter,
|
||||
|
|
@ -41,6 +40,7 @@ import {
|
|||
useStatusFlag,
|
||||
useWROLMode
|
||||
} from "../hooks/customHooks";
|
||||
import {useFileWorkerStatus} from "../contexts/FileWorkerStatusContext";
|
||||
import {Route, Routes} from "react-router";
|
||||
import {CardPlaceholder} from "./Placeholder";
|
||||
import {ArchiveCard, ArchiveRowCells} from "./Archive";
|
||||
|
|
@ -566,7 +566,7 @@ function getPhaseLabel(status) {
|
|||
}
|
||||
|
||||
export function FilesRefreshProgress() {
|
||||
const {progress} = useFilesProgressInterval();
|
||||
const {status: progress} = useFileWorkerStatus();
|
||||
|
||||
if (!progress || progress.status === 'idle') {
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/c
|
|||
import {DarkModeToggle, HotspotStatusIcon, useLocalStorage} from "./Common";
|
||||
import {ShareButton} from "./Share";
|
||||
import {useCPUTemperature, useIOStats, useLoad, useMemoryStats, usePowerStats, useWROLMode} from "../hooks/customHooks";
|
||||
import {useReorganizationStatus} from "../contexts/FileWorkerStatusContext";
|
||||
import {SearchIconButton} from "./Search";
|
||||
import {Icon, Popup} from "./Theme";
|
||||
import {HELP_VIEWER_URI, NAME} from "./Vars";
|
||||
|
|
@ -165,10 +166,33 @@ export function NavBar() {
|
|||
// issue.
|
||||
const warningIcon = temperatureIcon || diskWaitIcon || memoryIcon || systemLoadIcon;
|
||||
|
||||
const {isReorganizing, taskType, collectionId, collectionKind} = useReorganizationStatus();
|
||||
|
||||
let processingLink;
|
||||
if (status && status.flags) {
|
||||
if (status.flags.file_worker_busy) {
|
||||
processingLink = '/files';
|
||||
// Smart navigation based on what the file worker is doing
|
||||
if (isReorganizing && collectionId && collectionKind) {
|
||||
// Single collection reorganization - navigate to that collection
|
||||
if (collectionKind === 'channel') {
|
||||
processingLink = `/videos/channel/${collectionId}/edit`;
|
||||
} else if (collectionKind === 'domain') {
|
||||
processingLink = `/archive/domain/${collectionId}/edit`;
|
||||
} else {
|
||||
processingLink = '/files';
|
||||
}
|
||||
} else if (isReorganizing && taskType === 'batch_reorganize' && collectionKind) {
|
||||
// Batch reorganization - navigate to settings page for that kind
|
||||
if (collectionKind === 'channel') {
|
||||
processingLink = '/videos/settings';
|
||||
} else if (collectionKind === 'domain') {
|
||||
processingLink = '/archive/settings';
|
||||
} else {
|
||||
processingLink = '/files';
|
||||
}
|
||||
} else {
|
||||
processingLink = '/files';
|
||||
}
|
||||
} else if (status.flags.map_importing) {
|
||||
processingLink = '/map/manage';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@ import _ from "lodash";
|
|||
import {defaultFileOrder, defaultSearchOrder, HELP_VIEWER_URI} from "./Vars";
|
||||
import {InputForm, ToggleForm, useForm} from "../hooks/useForm";
|
||||
import {VideoResolutionSelectorForm} from "./Download";
|
||||
import {BatchReorganizeModal} from "./collections/BatchReorganizeModal";
|
||||
import {previewBatchReorganization} from "../api";
|
||||
|
||||
export function VideoWrapper() {
|
||||
const {videoId} = useParams();
|
||||
|
|
@ -234,15 +236,25 @@ function VideosPage() {
|
|||
function VideoFileNameForm({form}) {
|
||||
const [message, setMessage] = React.useState(null);
|
||||
|
||||
const onChange = async (value) => {
|
||||
const response = await postVideoFileFormat(value);
|
||||
const {error, preview} = await response.json();
|
||||
if (error) {
|
||||
setMessage({content: error, header: 'Invalid File Name', negative: true});
|
||||
} else {
|
||||
setMessage({content: preview, header: 'File Name Preview', positive: true});
|
||||
}
|
||||
}
|
||||
const debouncedOnChange = React.useCallback(
|
||||
_.debounce(async (value) => {
|
||||
const response = await postVideoFileFormat(value);
|
||||
const {error, preview} = await response.json();
|
||||
if (error) {
|
||||
setMessage({content: error, header: 'Invalid File Name', negative: true});
|
||||
} else {
|
||||
setMessage({content: preview, header: 'File Name Preview', positive: true});
|
||||
}
|
||||
}, 300),
|
||||
[]
|
||||
);
|
||||
|
||||
// Cleanup debounce on unmount
|
||||
React.useEffect(() => {
|
||||
return () => debouncedOnChange.cancel();
|
||||
}, [debouncedOnChange]);
|
||||
|
||||
const onChange = debouncedOnChange;
|
||||
|
||||
const label = <InfoHeader
|
||||
headerSize='h5'
|
||||
|
|
@ -257,10 +269,10 @@ function VideoFileNameForm({form}) {
|
|||
<li><code>%(upload_year)s</code> - Upload year (YYYY)</li>
|
||||
<li><code>%(upload_month)s</code> - Upload month (01-12)</li>
|
||||
<li><code>%(id)s</code> - Video ID</li>
|
||||
<li><code>%(ext)s</code> - File extension</li>
|
||||
<li><code>%(channel)s</code> - Channel name</li>
|
||||
<li><code>%(duration)s</code> - Duration in seconds</li>
|
||||
<li><code>%(playlist_index)s</code> - Playlist index</li>
|
||||
<li><code>%(ext)s</code> - File extension (required, must be at end)</li>
|
||||
</ul>
|
||||
<p>See yt-dlp docs for all variables. Subdirectories supported.</p>
|
||||
</>}
|
||||
|
|
@ -279,6 +291,25 @@ function VideoFileNameForm({form}) {
|
|||
function VideosSettingsPage() {
|
||||
useTitle('Videos Settings');
|
||||
|
||||
const [batchModalOpen, setBatchModalOpen] = useState(false);
|
||||
const [channelsNeedingReorg, setChannelsNeedingReorg] = useState(0);
|
||||
const [fetchingReorgCount, setFetchingReorgCount] = useState(true);
|
||||
|
||||
// Check how many channels need reorganization on mount
|
||||
useEffect(() => {
|
||||
setFetchingReorgCount(true);
|
||||
previewBatchReorganization('channel')
|
||||
.then(data => {
|
||||
setChannelsNeedingReorg(data.total_collections || 0);
|
||||
})
|
||||
.catch(() => {
|
||||
setChannelsNeedingReorg(0);
|
||||
})
|
||||
.finally(() => {
|
||||
setFetchingReorgCount(false);
|
||||
});
|
||||
}, []);
|
||||
|
||||
const emptyFormData = {
|
||||
video_resolutions: ['1080p', '720p', '480p', 'maximum'],
|
||||
yt_dlp_options: {
|
||||
|
|
@ -337,10 +368,11 @@ function VideosSettingsPage() {
|
|||
popupContent='Always download videos with the selected browser profile. This is risky, and therefore discouraged.'
|
||||
/>;
|
||||
|
||||
return <Segment>
|
||||
<Header as='h3'>Video Downloader Config</Header>
|
||||
return <>
|
||||
<Segment>
|
||||
<Header as='h3'>Video Downloader Config</Header>
|
||||
|
||||
<Form>
|
||||
<Form>
|
||||
<Grid>
|
||||
<Grid.Row columns={2}>
|
||||
<Grid.Column mobile={16} computer={8}>
|
||||
|
|
@ -435,7 +467,42 @@ function VideosSettingsPage() {
|
|||
</Grid.Row>
|
||||
</Grid>
|
||||
</Form>
|
||||
</Segment>
|
||||
</Segment>
|
||||
|
||||
<Segment>
|
||||
<Header as='h4'>File Organization</Header>
|
||||
<p>
|
||||
{fetchingReorgCount
|
||||
? 'Checking for channels that need reorganization...'
|
||||
: <>
|
||||
<strong>{channelsNeedingReorg}</strong> channel{channelsNeedingReorg !== 1 ? 's' : ''}
|
||||
{channelsNeedingReorg > 0
|
||||
? ' have files that do not match the current file name format.'
|
||||
: '. All channels are organized correctly.'}
|
||||
</>
|
||||
}
|
||||
</p>
|
||||
<Button
|
||||
color='orange'
|
||||
onClick={() => setBatchModalOpen(true)}
|
||||
id='reorganize_all_channels_button'
|
||||
disabled={fetchingReorgCount || channelsNeedingReorg === 0}
|
||||
loading={fetchingReorgCount}
|
||||
>
|
||||
<Icon name='folder open outline'/> Reorganize All Channels
|
||||
</Button>
|
||||
</Segment>
|
||||
|
||||
<BatchReorganizeModal
|
||||
open={batchModalOpen}
|
||||
onClose={() => setBatchModalOpen(false)}
|
||||
kind='channel'
|
||||
onComplete={() => {
|
||||
setBatchModalOpen(false);
|
||||
setChannelsNeedingReorg(0);
|
||||
}}
|
||||
/>
|
||||
</>
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -303,15 +303,15 @@ export function StatusPage() {
|
|||
{cpuProgress}
|
||||
{memoryUsageProgress}
|
||||
<Statistic.Group size='mini'>
|
||||
<LoadStatistic label='1 Minute Load' value={minute_1} cores={cores}/>
|
||||
<LoadStatistic label='5 Minute Load' value={minute_5} cores={cores}/>
|
||||
<LoadStatistic label='15 Minute Load' value={minute_15} cores={cores}/>
|
||||
<CPUTemperatureStatistic
|
||||
id='cpu_temperature_statistic'
|
||||
temperature={temperature}
|
||||
high_temperature={high_temperature}
|
||||
critical_temperature={critical_temperature}
|
||||
/>
|
||||
<LoadStatistic label='1 Minute Load' value={minute_1} cores={cores}/>
|
||||
<LoadStatistic label='5 Minute Load' value={minute_5} cores={cores}/>
|
||||
<LoadStatistic label='15 Minute Load' value={minute_15} cores={cores}/>
|
||||
<IOWaitStatistic percent_iowait={percent_iowait}/>
|
||||
<UptimeStatistic uptime_seconds={uptime_seconds}/>
|
||||
</Statistic.Group>
|
||||
|
|
@ -322,6 +322,9 @@ export function StatusPage() {
|
|||
{cpuProgress}
|
||||
{memoryUsageProgress}
|
||||
<Statistic.Group>
|
||||
<LoadStatistic label='1 Min. Load' value={minute_1} cores={cores}/>
|
||||
<LoadStatistic label='5 Min.' value={minute_5} cores={cores}/>
|
||||
<LoadStatistic label='15 Min.' value={minute_15} cores={cores}/>
|
||||
<CPUTemperatureStatistic
|
||||
id='cpu_temperature_statistic'
|
||||
temperature={temperature}
|
||||
|
|
@ -329,9 +332,6 @@ export function StatusPage() {
|
|||
critical_temperature={critical_temperature}
|
||||
style={{marginRight: 0}}
|
||||
/>
|
||||
<LoadStatistic label='1 Min. Load' value={minute_1} cores={cores}/>
|
||||
<LoadStatistic label='5 Min.' value={minute_5} cores={cores}/>
|
||||
<LoadStatistic label='15 Min.' value={minute_15} cores={cores}/>
|
||||
<IOWaitStatistic percent_iowait={percent_iowait}/>
|
||||
<UptimeStatistic uptime_seconds={uptime_seconds}/>
|
||||
</Statistic.Group>
|
||||
|
|
|
|||
411
app/src/components/collections/BatchReorganizeModal.js
Normal file
411
app/src/components/collections/BatchReorganizeModal.js
Normal file
|
|
@ -0,0 +1,411 @@
|
|||
import React, {useCallback, useEffect, useState} from 'react';
|
||||
import {Grid, Icon, Label, Progress} from 'semantic-ui-react';
|
||||
import Message from 'semantic-ui-react/dist/commonjs/collections/Message';
|
||||
import {Button, Modal, Table} from '../Theme';
|
||||
import {APIButton} from '../Common';
|
||||
import {
|
||||
executeBatchReorganization,
|
||||
getBatchReorganizationStatus,
|
||||
previewBatchReorganization
|
||||
} from '../../api';
|
||||
import {useReorganizationStatus, useFileWorkerStatus} from '../../contexts/FileWorkerStatusContext';
|
||||
|
||||
/**
|
||||
* Modal component for previewing and executing batch collection file reorganization.
|
||||
*
|
||||
* Shows a list of collections that need reorganization, allows user to execute
|
||||
* batch reorganization, and tracks progress with dual progress bars (overall and per-collection).
|
||||
*
|
||||
* @param {boolean} open - Whether the modal is open
|
||||
* @param {Function} onClose - Callback when modal is closed
|
||||
* @param {string} kind - 'channel' or 'domain'
|
||||
* @param {Function} onComplete - Optional callback when reorganization completes
|
||||
*/
|
||||
export function BatchReorganizeModal({
|
||||
open,
|
||||
onClose,
|
||||
kind,
|
||||
onComplete,
|
||||
}) {
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [preview, setPreview] = useState(null);
|
||||
const [error, setError] = useState(null);
|
||||
const [batchJobId, setBatchJobId] = useState(null);
|
||||
const [status, setStatus] = useState(null);
|
||||
const [polling, setPolling] = useState(false);
|
||||
|
||||
const kindLabel = kind === 'channel' ? 'Channel' : 'Domain';
|
||||
const kindLabelPlural = kind === 'channel' ? 'Channels' : 'Domains';
|
||||
|
||||
// Check if batch reorganization is currently active
|
||||
const {
|
||||
isReorganizing,
|
||||
taskType,
|
||||
collectionKind,
|
||||
batchStatus: workerBatchStatus,
|
||||
workerStatus,
|
||||
} = useReorganizationStatus();
|
||||
const {setFastPolling} = useFileWorkerStatus();
|
||||
|
||||
// Enable fast polling when modal is open
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setFastPolling(true);
|
||||
}
|
||||
return () => setFastPolling(false);
|
||||
}, [open, setFastPolling]);
|
||||
|
||||
// Load preview when modal opens, or resume if already reorganizing
|
||||
useEffect(() => {
|
||||
if (open && kind) {
|
||||
// Check if batch_status has an active job for this kind
|
||||
// This is the primary indicator since worker status can briefly show
|
||||
// 'reorganizing'/'reorganize' during individual collection processing
|
||||
const hasActiveBatchJob = workerBatchStatus?.batch_job_id &&
|
||||
collectionKind === kind;
|
||||
|
||||
// Worker is actively doing something (taskType is set)
|
||||
// After batch completes, taskType becomes null, so this distinguishes
|
||||
// "batch in progress" from "batch completed but batch_job_id still cached"
|
||||
const workerIsActive = taskType != null;
|
||||
|
||||
// Use either the standard detection OR the batch_status fallback (only when worker is active)
|
||||
const isThisActive = (isReorganizing && taskType === 'batch_reorganize' && collectionKind === kind) ||
|
||||
(hasActiveBatchJob && workerIsActive);
|
||||
|
||||
if (isThisActive && workerBatchStatus) {
|
||||
// Resume: start polling API immediately for accurate progress
|
||||
// Don't use workerBatchStatus.current_collection directly as it may have stale
|
||||
// percent/total/completed values. The batch status API returns fresh values
|
||||
// by copying operation_percent into current_collection.percent.
|
||||
setLoading(false);
|
||||
setPreview(null);
|
||||
setBatchJobId(workerBatchStatus.batch_job_id || null);
|
||||
// Don't setStatus from workerBatchStatus - let the first poll populate it
|
||||
// with accurate values from the batch status API
|
||||
setPolling(true);
|
||||
} else if (!polling && !batchJobId) {
|
||||
// Normal: fetch preview (only if not already polling/resuming)
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
setPreview(null);
|
||||
setBatchJobId(null);
|
||||
setStatus(null);
|
||||
|
||||
previewBatchReorganization(kind)
|
||||
.then(data => {
|
||||
setPreview(data);
|
||||
setLoading(false);
|
||||
})
|
||||
.catch(err => {
|
||||
setError(err.message);
|
||||
setLoading(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [open, kind, isReorganizing, taskType, collectionKind]);
|
||||
|
||||
// Poll for status when we have a batch job ID
|
||||
useEffect(() => {
|
||||
if (!batchJobId || !polling) {
|
||||
return;
|
||||
}
|
||||
|
||||
let timeoutId = null;
|
||||
let cancelled = false;
|
||||
|
||||
const pollStatus = async () => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const statusData = await getBatchReorganizationStatus(batchJobId);
|
||||
if (cancelled) return;
|
||||
|
||||
setStatus(statusData);
|
||||
|
||||
if (statusData.status === 'complete') {
|
||||
setPolling(false);
|
||||
if (onComplete) {
|
||||
onComplete();
|
||||
}
|
||||
} else if (statusData.status === 'failed' || statusData.error) {
|
||||
setPolling(false);
|
||||
setError(statusData.error || 'Batch reorganization failed');
|
||||
} else {
|
||||
// Schedule next poll after this one completes
|
||||
timeoutId = setTimeout(pollStatus, 500);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to get batch reorganization status', err);
|
||||
if (!cancelled) {
|
||||
// Retry after delay even on error
|
||||
timeoutId = setTimeout(pollStatus, 1000);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Start polling immediately
|
||||
pollStatus();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
};
|
||||
}, [batchJobId, polling, onComplete]);
|
||||
|
||||
const handleClose = useCallback(() => {
|
||||
// Allow closing - user can reopen to resume if needed
|
||||
setPreview(null);
|
||||
setError(null);
|
||||
setBatchJobId(null);
|
||||
setStatus(null);
|
||||
setPolling(false);
|
||||
onClose();
|
||||
}, [onClose]);
|
||||
|
||||
const handleReorganize = useCallback(async () => {
|
||||
try {
|
||||
setError(null);
|
||||
const result = await executeBatchReorganization(kind);
|
||||
if (result.batch_job_id) {
|
||||
setBatchJobId(result.batch_job_id);
|
||||
setPolling(true);
|
||||
setStatus({
|
||||
status: 'pending',
|
||||
total_collections: result.collection_count,
|
||||
completed_collections: 0,
|
||||
current_collection: null,
|
||||
overall_percent: 0,
|
||||
completed: [],
|
||||
failed_collection: null,
|
||||
error: null,
|
||||
});
|
||||
} else {
|
||||
// No collections needed reorganization
|
||||
if (onComplete) {
|
||||
onComplete();
|
||||
}
|
||||
handleClose();
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err.message);
|
||||
}
|
||||
}, [kind, onComplete, handleClose]);
|
||||
|
||||
const renderPreview = () => {
|
||||
if (!preview) return null;
|
||||
|
||||
return (
|
||||
<Grid columns={1}>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<p>
|
||||
<strong>{preview.total_collections}</strong> {kindLabelPlural.toLowerCase()} need reorganization,
|
||||
with <strong>{preview.total_files_needing_move}</strong> total files to move.
|
||||
</p>
|
||||
<p>
|
||||
<strong>New format:</strong> <Label>{preview.new_file_format}</Label>
|
||||
</p>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
|
||||
{preview.collections && preview.collections.length > 0 && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<strong>{kindLabelPlural} to reorganize:</strong>
|
||||
<Table basic='very' compact size='small'>
|
||||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell>{kindLabel}</Table.HeaderCell>
|
||||
<Table.HeaderCell>Files to Move</Table.HeaderCell>
|
||||
<Table.HeaderCell>Sample</Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
<Table.Body>
|
||||
{preview.collections.slice(0, 10).map((collection, idx) => (
|
||||
<Table.Row key={idx}>
|
||||
<Table.Cell>{collection.collection_name}</Table.Cell>
|
||||
<Table.Cell>
|
||||
{collection.files_needing_move} / {collection.total_files}
|
||||
</Table.Cell>
|
||||
<Table.Cell style={{wordBreak: 'break-all', fontSize: '0.9em'}}>
|
||||
{collection.sample_move ? (
|
||||
<>
|
||||
<span style={{color: '#888'}}>{collection.sample_move.old_path}</span>
|
||||
<br/>
|
||||
<Icon name='arrow right' size='small'/>
|
||||
<span>{collection.sample_move.new_path}</span>
|
||||
</>
|
||||
) : '-'}
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
))}
|
||||
</Table.Body>
|
||||
</Table>
|
||||
{preview.collections.length > 10 && (
|
||||
<p style={{fontStyle: 'italic'}}>
|
||||
...and {preview.collections.length - 10} more {kindLabelPlural.toLowerCase()}
|
||||
</p>
|
||||
)}
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
|
||||
{preview.total_collections === 0 && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Message info>
|
||||
<Message.Header>No {kindLabelPlural} Need Reorganization</Message.Header>
|
||||
<p>All {kindLabelPlural.toLowerCase()} are already organized according to the current format.</p>
|
||||
</Message>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
</Grid>
|
||||
);
|
||||
};
|
||||
|
||||
const renderProgress = () => {
|
||||
if (!status) return null;
|
||||
|
||||
const currentCollection = status.current_collection;
|
||||
|
||||
return (
|
||||
<Grid columns={1}>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<p>
|
||||
<strong>Overall Status:</strong> {status.status}
|
||||
</p>
|
||||
<strong>Overall Progress:</strong>
|
||||
<Progress
|
||||
percent={status.overall_percent || 0}
|
||||
progress
|
||||
indicating={status.status !== 'complete' && status.status !== 'failed'}
|
||||
success={status.status === 'complete'}
|
||||
error={status.status === 'failed'}
|
||||
/>
|
||||
<p>
|
||||
{status.completed_collections || 0} of {status.total_collections || 0} {kindLabelPlural.toLowerCase()} completed
|
||||
</p>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
|
||||
{currentCollection && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<strong>Currently Processing:</strong> {currentCollection.name}
|
||||
<Progress
|
||||
percent={currentCollection.percent || 0}
|
||||
progress
|
||||
indicating
|
||||
size='small'
|
||||
color='blue'
|
||||
/>
|
||||
<p style={{fontSize: '0.9em'}}>
|
||||
{currentCollection.completed || 0} of {currentCollection.total || 0} files
|
||||
</p>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
|
||||
{status.completed && status.completed.length > 0 && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<strong>Completed:</strong>
|
||||
<div style={{maxHeight: '150px', overflowY: 'auto'}}>
|
||||
{status.completed.map((c, idx) => (
|
||||
<Label key={idx} color='green' style={{margin: '2px'}}>
|
||||
<Icon name='check'/> {c.name}
|
||||
</Label>
|
||||
))}
|
||||
</div>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
|
||||
{status.status === 'complete' && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Message success>
|
||||
<Message.Header>Batch Reorganization Complete</Message.Header>
|
||||
<p>All {kindLabelPlural.toLowerCase()} have been reorganized successfully.</p>
|
||||
</Message>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
|
||||
{status.status === 'failed' && status.failed_collection && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Message negative>
|
||||
<Message.Header>Reorganization Failed</Message.Header>
|
||||
<p>
|
||||
Failed on {kindLabel.toLowerCase()}: <strong>{status.failed_collection.name}</strong>
|
||||
</p>
|
||||
{status.error && <p>Error: {status.error}</p>}
|
||||
</Message>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
</Grid>
|
||||
);
|
||||
};
|
||||
|
||||
const isInProgress = polling || (status && status.status !== 'complete' && status.status !== 'failed');
|
||||
const canReorganize = preview && preview.total_collections > 0 && !isInProgress && !batchJobId;
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open={open}
|
||||
onClose={handleClose}
|
||||
closeIcon={true}
|
||||
closeOnDimmerClick={!isInProgress}
|
||||
closeOnEscape={!isInProgress}
|
||||
size='large'
|
||||
>
|
||||
<Modal.Header>
|
||||
Reorganize All {kindLabelPlural}
|
||||
</Modal.Header>
|
||||
<Modal.Content scrolling>
|
||||
{loading && (
|
||||
<Message icon>
|
||||
<Icon name='circle notched' loading/>
|
||||
<Message.Content>
|
||||
<Message.Header>Loading Preview</Message.Header>
|
||||
Analyzing {kindLabelPlural.toLowerCase()}...
|
||||
</Message.Content>
|
||||
</Message>
|
||||
)}
|
||||
|
||||
{error && !status?.failed_collection && (
|
||||
<Message negative>
|
||||
<Message.Header>Error</Message.Header>
|
||||
<p>{error}</p>
|
||||
</Message>
|
||||
)}
|
||||
|
||||
{!loading && !batchJobId && renderPreview()}
|
||||
{batchJobId && renderProgress()}
|
||||
</Modal.Content>
|
||||
<Modal.Actions>
|
||||
<Button onClick={handleClose}>
|
||||
{status?.status === 'complete' ? 'Close' : isInProgress ? 'Hide' : 'Cancel'}
|
||||
</Button>
|
||||
{canReorganize && (
|
||||
<APIButton
|
||||
color='violet'
|
||||
onClick={handleReorganize}
|
||||
obeyWROLMode={true}
|
||||
>
|
||||
Reorganize All {kindLabelPlural}
|
||||
</APIButton>
|
||||
)}
|
||||
</Modal.Actions>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
316
app/src/components/collections/BatchReorganizeModal.test.js
Normal file
316
app/src/components/collections/BatchReorganizeModal.test.js
Normal file
|
|
@ -0,0 +1,316 @@
|
|||
import React from 'react';
|
||||
import {screen, waitFor, fireEvent} from '@testing-library/react';
|
||||
import {renderWithProviders} from '../../test-utils';
|
||||
import {BatchReorganizeModal} from './BatchReorganizeModal';
|
||||
import * as api from '../../api';
|
||||
|
||||
// Mock the API functions
|
||||
jest.mock('../../api', () => ({
|
||||
previewBatchReorganization: jest.fn(),
|
||||
executeBatchReorganization: jest.fn(),
|
||||
getBatchReorganizationStatus: jest.fn(),
|
||||
}));
|
||||
|
||||
// Mock react-semantic-toasts-2
|
||||
jest.mock('react-semantic-toasts-2', () => ({
|
||||
toast: jest.fn(),
|
||||
}));
|
||||
|
||||
// Mock the FileWorkerStatusContext
|
||||
jest.mock('../../contexts/FileWorkerStatusContext', () => ({
|
||||
useReorganizationStatus: () => ({
|
||||
isReorganizing: false,
|
||||
taskType: null,
|
||||
collectionId: null,
|
||||
collectionKind: null,
|
||||
batchStatus: null,
|
||||
workerStatus: null,
|
||||
refresh: jest.fn(),
|
||||
}),
|
||||
useFileWorkerStatus: () => ({
|
||||
status: null,
|
||||
error: null,
|
||||
refresh: jest.fn(),
|
||||
setFastPolling: jest.fn(),
|
||||
}),
|
||||
}));
|
||||
|
||||
describe('BatchReorganizeModal', () => {
|
||||
const defaultProps = {
|
||||
open: true,
|
||||
onClose: jest.fn(),
|
||||
kind: 'channel',
|
||||
onComplete: jest.fn(),
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('loads preview when opened', async () => {
|
||||
const mockPreview = {
|
||||
collections: [
|
||||
{
|
||||
collection_id: 1,
|
||||
collection_name: 'Test Channel',
|
||||
total_files: 50,
|
||||
files_needing_move: 45,
|
||||
sample_move: {old_path: 'old/path.mp4', new_path: 'new/path.mp4'},
|
||||
},
|
||||
],
|
||||
total_collections: 1,
|
||||
total_files_needing_move: 45,
|
||||
new_file_format: '%(upload_date)s/%(title)s.%(ext)s',
|
||||
};
|
||||
|
||||
api.previewBatchReorganization.mockResolvedValue(mockPreview);
|
||||
|
||||
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
|
||||
|
||||
// Should show loading initially
|
||||
expect(screen.getByText(/Loading Preview/i)).toBeInTheDocument();
|
||||
|
||||
// Wait for preview to load
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Test Channel/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
expect(api.previewBatchReorganization).toHaveBeenCalledWith('channel');
|
||||
});
|
||||
|
||||
it('shows collection list with sample moves', async () => {
|
||||
const mockPreview = {
|
||||
collections: [
|
||||
{
|
||||
collection_id: 1,
|
||||
collection_name: 'Channel A',
|
||||
total_files: 100,
|
||||
files_needing_move: 80,
|
||||
sample_move: {old_path: 'videos/old.mp4', new_path: 'videos/new.mp4'},
|
||||
},
|
||||
{
|
||||
collection_id: 2,
|
||||
collection_name: 'Channel B',
|
||||
total_files: 50,
|
||||
files_needing_move: 30,
|
||||
sample_move: {old_path: 'videos/old2.mp4', new_path: 'videos/new2.mp4'},
|
||||
},
|
||||
],
|
||||
total_collections: 2,
|
||||
total_files_needing_move: 110,
|
||||
new_file_format: '%(upload_date)s/%(title)s.%(ext)s',
|
||||
};
|
||||
|
||||
api.previewBatchReorganization.mockResolvedValue(mockPreview);
|
||||
|
||||
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('Channel A')).toBeInTheDocument();
|
||||
expect(screen.getByText('Channel B')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Should show total statistics - look for the numbers in the text context
|
||||
expect(screen.getByText('2')).toBeInTheDocument(); // total collections
|
||||
expect(screen.getByText('110')).toBeInTheDocument(); // total files
|
||||
});
|
||||
|
||||
it('shows total statistics', async () => {
|
||||
const mockPreview = {
|
||||
collections: [
|
||||
{collection_id: 1, collection_name: 'ChannelA', total_files: 50, files_needing_move: 45, sample_move: null},
|
||||
{collection_id: 2, collection_name: 'ChannelB', total_files: 30, files_needing_move: 25, sample_move: null},
|
||||
{collection_id: 3, collection_name: 'ChannelC', total_files: 20, files_needing_move: 15, sample_move: null},
|
||||
],
|
||||
total_collections: 3,
|
||||
total_files_needing_move: 85,
|
||||
new_file_format: '%(title)s.%(ext)s',
|
||||
};
|
||||
|
||||
api.previewBatchReorganization.mockResolvedValue(mockPreview);
|
||||
|
||||
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
|
||||
|
||||
await waitFor(() => {
|
||||
// Look for channel names to verify list is displayed
|
||||
expect(screen.getByText('ChannelA')).toBeInTheDocument();
|
||||
expect(screen.getByText('ChannelB')).toBeInTheDocument();
|
||||
expect(screen.getByText('ChannelC')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('executes batch reorganization on button click', async () => {
|
||||
const mockPreview = {
|
||||
collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}],
|
||||
total_collections: 1,
|
||||
total_files_needing_move: 5,
|
||||
new_file_format: '%(title)s.%(ext)s',
|
||||
};
|
||||
|
||||
api.previewBatchReorganization.mockResolvedValue(mockPreview);
|
||||
api.executeBatchReorganization.mockResolvedValue({
|
||||
batch_job_id: 'batch-reorganize-abc123',
|
||||
message: 'Batch reorganization started',
|
||||
collection_count: 1,
|
||||
});
|
||||
api.getBatchReorganizationStatus.mockResolvedValue({
|
||||
status: 'complete',
|
||||
total_collections: 1,
|
||||
completed_collections: 1,
|
||||
current_collection: null,
|
||||
overall_percent: 100,
|
||||
completed: [{id: 1, name: 'Test', status: 'complete'}],
|
||||
failed_collection: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
|
||||
|
||||
// Wait for preview to load
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Click the reorganize button
|
||||
fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i}));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(api.executeBatchReorganization).toHaveBeenCalledWith('channel');
|
||||
});
|
||||
});
|
||||
|
||||
it('shows overall progress bar during execution', async () => {
|
||||
const mockPreview = {
|
||||
collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}],
|
||||
total_collections: 1,
|
||||
total_files_needing_move: 5,
|
||||
new_file_format: '%(title)s.%(ext)s',
|
||||
};
|
||||
|
||||
api.previewBatchReorganization.mockResolvedValue(mockPreview);
|
||||
api.executeBatchReorganization.mockResolvedValue({
|
||||
batch_job_id: 'batch-reorganize-abc123',
|
||||
message: 'Batch reorganization started',
|
||||
collection_count: 1,
|
||||
});
|
||||
api.getBatchReorganizationStatus.mockResolvedValue({
|
||||
status: 'running',
|
||||
total_collections: 1,
|
||||
completed_collections: 0,
|
||||
current_collection: {id: 1, name: 'Test', status: 'running', total: 5, completed: 2, percent: 40},
|
||||
overall_percent: 40,
|
||||
completed: [],
|
||||
failed_collection: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument();
|
||||
});
|
||||
|
||||
fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i}));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Overall Progress/i)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('shows per-collection progress bars', async () => {
|
||||
const mockPreview = {
|
||||
collections: [{collection_id: 1, collection_name: 'Test Channel', total_files: 10, files_needing_move: 5, sample_move: null}],
|
||||
total_collections: 1,
|
||||
total_files_needing_move: 5,
|
||||
new_file_format: '%(title)s.%(ext)s',
|
||||
};
|
||||
|
||||
api.previewBatchReorganization.mockResolvedValue(mockPreview);
|
||||
api.executeBatchReorganization.mockResolvedValue({
|
||||
batch_job_id: 'batch-reorganize-abc123',
|
||||
message: 'Batch reorganization started',
|
||||
collection_count: 1,
|
||||
});
|
||||
api.getBatchReorganizationStatus.mockResolvedValue({
|
||||
status: 'running',
|
||||
total_collections: 1,
|
||||
completed_collections: 0,
|
||||
current_collection: {id: 1, name: 'Test Channel', status: 'running', total: 5, completed: 2, percent: 40},
|
||||
overall_percent: 20,
|
||||
completed: [],
|
||||
failed_collection: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
renderWithProviders(<BatchReorganizeModal {...defaultProps} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument();
|
||||
});
|
||||
|
||||
fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i}));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Currently Processing/i)).toBeInTheDocument();
|
||||
expect(screen.getByText(/Test Channel/)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('calls onComplete when finished', async () => {
|
||||
const mockOnComplete = jest.fn();
|
||||
const mockPreview = {
|
||||
collections: [{collection_id: 1, collection_name: 'Test', total_files: 10, files_needing_move: 5, sample_move: null}],
|
||||
total_collections: 1,
|
||||
total_files_needing_move: 5,
|
||||
new_file_format: '%(title)s.%(ext)s',
|
||||
};
|
||||
|
||||
api.previewBatchReorganization.mockResolvedValue(mockPreview);
|
||||
api.executeBatchReorganization.mockResolvedValue({
|
||||
batch_job_id: 'batch-reorganize-abc123',
|
||||
message: 'Batch reorganization started',
|
||||
collection_count: 1,
|
||||
});
|
||||
api.getBatchReorganizationStatus.mockResolvedValue({
|
||||
status: 'complete',
|
||||
total_collections: 1,
|
||||
completed_collections: 1,
|
||||
current_collection: null,
|
||||
overall_percent: 100,
|
||||
completed: [{id: 1, name: 'Test', status: 'complete'}],
|
||||
failed_collection: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
renderWithProviders(<BatchReorganizeModal {...defaultProps} onComplete={mockOnComplete} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole('button', {name: /Reorganize All Channels/i})).toBeInTheDocument();
|
||||
});
|
||||
|
||||
fireEvent.click(screen.getByRole('button', {name: /Reorganize All Channels/i}));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockOnComplete).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('handles domain kind correctly', async () => {
|
||||
const mockPreview = {
|
||||
collections: [{collection_id: 1, collection_name: 'example.com', total_files: 10, files_needing_move: 5, sample_move: null}],
|
||||
total_collections: 1,
|
||||
total_files_needing_move: 5,
|
||||
new_file_format: '%(title)s.%(ext)s',
|
||||
};
|
||||
|
||||
api.previewBatchReorganization.mockResolvedValue(mockPreview);
|
||||
|
||||
renderWithProviders(<BatchReorganizeModal {...defaultProps} kind="domain" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Reorganize All Domains/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
expect(api.previewBatchReorganization).toHaveBeenCalledWith('domain');
|
||||
});
|
||||
});
|
||||
345
app/src/components/collections/CollectionReorganizeModal.js
Normal file
345
app/src/components/collections/CollectionReorganizeModal.js
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
import React, {useCallback, useEffect, useState} from 'react';
|
||||
import {Grid, Icon, Label, Progress} from 'semantic-ui-react';
|
||||
import Message from 'semantic-ui-react/dist/commonjs/collections/Message';
|
||||
import {Button, Modal, Table} from '../Theme';
|
||||
import {APIButton} from '../Common';
|
||||
import {
|
||||
executeCollectionReorganization,
|
||||
getReorganizationStatus,
|
||||
previewCollectionReorganization
|
||||
} from '../../api';
|
||||
import {useReorganizationStatus} from '../../contexts/FileWorkerStatusContext';
|
||||
|
||||
/**
|
||||
* Modal component for previewing and executing collection file reorganization.
|
||||
*
|
||||
* Shows a preview of files that will be moved when the file_name_format changes,
|
||||
* allows user to execute the reorganization, and tracks progress.
|
||||
*
|
||||
* @param {boolean} open - Whether the modal is open
|
||||
* @param {Function} onClose - Callback when modal is closed
|
||||
* @param {number} collectionId - ID of the collection to reorganize
|
||||
* @param {string} collectionName - Name of the collection for display
|
||||
* @param {Function} onComplete - Optional callback when reorganization completes
|
||||
* @param {boolean} needsReorganization - Whether the collection.needs_reorganization flag is true
|
||||
*/
|
||||
export function CollectionReorganizeModal({
|
||||
open,
|
||||
onClose,
|
||||
collectionId,
|
||||
collectionName,
|
||||
onComplete,
|
||||
needsReorganization,
|
||||
}) {
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [preview, setPreview] = useState(null);
|
||||
const [error, setError] = useState(null);
|
||||
const [jobId, setJobId] = useState(null);
|
||||
const [status, setStatus] = useState(null);
|
||||
const [polling, setPolling] = useState(false);
|
||||
|
||||
// Check if this collection is currently being reorganized
|
||||
const {
|
||||
isReorganizing,
|
||||
taskType,
|
||||
collectionId: activeCollectionId,
|
||||
workerStatus,
|
||||
} = useReorganizationStatus();
|
||||
|
||||
// Load preview when modal opens, or resume if already reorganizing
|
||||
useEffect(() => {
|
||||
if (open && collectionId) {
|
||||
// Check if THIS collection is currently being reorganized
|
||||
const isThisActive = isReorganizing &&
|
||||
taskType === 'reorganize' &&
|
||||
activeCollectionId === collectionId;
|
||||
|
||||
if (isThisActive) {
|
||||
// Resume: show current progress from worker status
|
||||
setLoading(false);
|
||||
setPreview(null);
|
||||
setJobId('active'); // Marker for active job
|
||||
setStatus({
|
||||
status: workerStatus.status,
|
||||
total: workerStatus.operation_total,
|
||||
completed: workerStatus.operation_processed,
|
||||
percent: workerStatus.operation_percent,
|
||||
});
|
||||
setPolling(true);
|
||||
} else if (!polling && !jobId) {
|
||||
// Normal: fetch preview (only if not already polling/resuming)
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
setPreview(null);
|
||||
setJobId(null);
|
||||
setStatus(null);
|
||||
|
||||
previewCollectionReorganization(collectionId)
|
||||
.then(data => {
|
||||
setPreview(data);
|
||||
setLoading(false);
|
||||
})
|
||||
.catch(err => {
|
||||
setError(err.message);
|
||||
setLoading(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [open, collectionId, isReorganizing, taskType, activeCollectionId]);
|
||||
|
||||
// Poll for status when we have a job ID
|
||||
// Uses setTimeout instead of setInterval to wait for each request to complete
|
||||
// before scheduling the next one, preventing request pile-up on slow connections
|
||||
useEffect(() => {
|
||||
if (!jobId || !polling) return;
|
||||
|
||||
let timeoutId = null;
|
||||
let cancelled = false;
|
||||
|
||||
const pollStatus = async () => {
|
||||
if (cancelled) return;
|
||||
|
||||
try {
|
||||
const statusData = await getReorganizationStatus(collectionId, jobId);
|
||||
if (cancelled) return;
|
||||
|
||||
setStatus(statusData);
|
||||
|
||||
if (statusData.status === 'complete') {
|
||||
setPolling(false);
|
||||
if (onComplete) {
|
||||
onComplete();
|
||||
}
|
||||
} else if (statusData.status === 'error' || statusData.error) {
|
||||
setPolling(false);
|
||||
setError(statusData.error || 'Reorganization failed');
|
||||
} else {
|
||||
// Schedule next poll after this one completes
|
||||
timeoutId = setTimeout(pollStatus, 1000);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to get reorganization status', err);
|
||||
if (!cancelled) {
|
||||
// Retry after delay even on error
|
||||
timeoutId = setTimeout(pollStatus, 1000);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Start polling immediately
|
||||
pollStatus();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
};
|
||||
}, [jobId, polling, collectionId, onComplete]);
|
||||
|
||||
const handleClose = useCallback(() => {
|
||||
// Allow closing - user can reopen to resume if needed
|
||||
setPreview(null);
|
||||
setError(null);
|
||||
setJobId(null);
|
||||
setStatus(null);
|
||||
setPolling(false);
|
||||
onClose();
|
||||
}, [onClose]);
|
||||
|
||||
const handleReorganize = useCallback(async () => {
|
||||
try {
|
||||
setError(null);
|
||||
const result = await executeCollectionReorganization(collectionId);
|
||||
if (result.job_id) {
|
||||
setJobId(result.job_id);
|
||||
setPolling(true);
|
||||
setStatus({status: 'pending', total: 0, completed: 0, percent: 0});
|
||||
} else {
|
||||
// No files needed reorganization
|
||||
if (onComplete) {
|
||||
onComplete();
|
||||
}
|
||||
handleClose();
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err.message);
|
||||
}
|
||||
}, [collectionId, onComplete, handleClose]);
|
||||
|
||||
const renderPreview = () => {
|
||||
if (!preview) return null;
|
||||
|
||||
return (
|
||||
<Grid columns={1}>
|
||||
{/* Warning when collection appears organized but has files to move */}
|
||||
{preview.files_needing_move > 0 && !needsReorganization && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Message warning>
|
||||
<Message.Header>Collection Appears Organized</Message.Header>
|
||||
<p>
|
||||
This collection's format matches the current configuration, but
|
||||
{' '}<strong>{preview.files_needing_move}</strong> files don't match the expected layout.
|
||||
This can happen if a previous reorganization was interrupted.
|
||||
</p>
|
||||
</Message>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<p>
|
||||
<strong>{preview.files_needing_move}</strong> of <strong>{preview.total_files}</strong> files
|
||||
will be reorganized.
|
||||
</p>
|
||||
{preview.current_file_format && (
|
||||
<p>
|
||||
<strong>Current format:</strong> <code>{preview.current_file_format}</code>
|
||||
</p>
|
||||
)}
|
||||
<p>
|
||||
<strong>New format:</strong> <Label>{preview.new_file_format}</Label>
|
||||
</p>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
|
||||
{preview.sample_moves && preview.sample_moves.length > 0 && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<strong>Sample moves:</strong>
|
||||
<Table basic='very' compact size='small'>
|
||||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell>Current Path</Table.HeaderCell>
|
||||
<Table.HeaderCell/>
|
||||
<Table.HeaderCell>New Path</Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
<Table.Body>
|
||||
{preview.sample_moves.map((move, idx) => (
|
||||
<Table.Row key={idx}>
|
||||
<Table.Cell style={{wordBreak: 'break-all'}}>
|
||||
{move.old_path}
|
||||
</Table.Cell>
|
||||
<Table.Cell>
|
||||
<Icon name='arrow right'/>
|
||||
</Table.Cell>
|
||||
<Table.Cell style={{wordBreak: 'break-all'}}>
|
||||
{move.new_path}
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
))}
|
||||
</Table.Body>
|
||||
</Table>
|
||||
{preview.files_needing_move > preview.sample_moves.length && (
|
||||
<p style={{fontStyle: 'italic'}}>
|
||||
...and {preview.files_needing_move - preview.sample_moves.length} more files
|
||||
</p>
|
||||
)}
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
|
||||
{preview.files_needing_move === 0 && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Message info>
|
||||
<Message.Header>No Files Need Reorganization</Message.Header>
|
||||
<p>All files are already organized according to the current format.</p>
|
||||
</Message>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
</Grid>
|
||||
);
|
||||
};
|
||||
|
||||
const renderProgress = () => {
|
||||
if (!status) return null;
|
||||
|
||||
return (
|
||||
<Grid columns={1}>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<p>
|
||||
<strong>Status:</strong> {status.status}
|
||||
</p>
|
||||
<Progress
|
||||
percent={status.percent || 0}
|
||||
progress
|
||||
indicating={status.status !== 'complete'}
|
||||
success={status.status === 'complete'}
|
||||
/>
|
||||
<p>
|
||||
{status.completed || 0} of {status.total || 0} files processed
|
||||
</p>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
|
||||
{status.status === 'complete' && (
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Message success>
|
||||
<Message.Header>Reorganization Complete</Message.Header>
|
||||
<p>All files have been reorganized successfully.</p>
|
||||
</Message>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
)}
|
||||
</Grid>
|
||||
);
|
||||
};
|
||||
|
||||
const isInProgress = polling || (status && status.status !== 'complete' && status.status !== 'error');
|
||||
const canReorganize = preview && preview.files_needing_move > 0 && !isInProgress && !jobId;
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open={open}
|
||||
onClose={handleClose}
|
||||
closeIcon={true}
|
||||
closeOnDimmerClick={!isInProgress}
|
||||
closeOnEscape={!isInProgress}
|
||||
>
|
||||
<Modal.Header>
|
||||
Reorganize Files: {collectionName}
|
||||
</Modal.Header>
|
||||
<Modal.Content>
|
||||
{loading && (
|
||||
<Message icon>
|
||||
<Icon name='circle notched' loading/>
|
||||
<Message.Content>
|
||||
<Message.Header>Loading Preview</Message.Header>
|
||||
Analyzing files...
|
||||
</Message.Content>
|
||||
</Message>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
<Message negative>
|
||||
<Message.Header>Error</Message.Header>
|
||||
<p>{error}</p>
|
||||
</Message>
|
||||
)}
|
||||
|
||||
{!loading && !jobId && renderPreview()}
|
||||
{jobId && renderProgress()}
|
||||
</Modal.Content>
|
||||
<Modal.Actions>
|
||||
<Button onClick={handleClose}>
|
||||
{status?.status === 'complete' ? 'Close' : isInProgress ? 'Hide' : 'Cancel'}
|
||||
</Button>
|
||||
{canReorganize && (
|
||||
<APIButton
|
||||
color='violet'
|
||||
onClick={handleReorganize}
|
||||
obeyWROLMode={true}
|
||||
>
|
||||
Reorganize Files
|
||||
</APIButton>
|
||||
)}
|
||||
</Modal.Actions>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
67
app/src/contexts/FileWorkerStatusContext.js
Normal file
67
app/src/contexts/FileWorkerStatusContext.js
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import React, {createContext, useContext, useState, useEffect, useCallback, useRef} from 'react';
|
||||
import {fetchFilesProgress} from '../api';
|
||||
|
||||
const FileWorkerStatusContext = createContext(null);
|
||||
|
||||
export function FileWorkerStatusProvider({children, pollInterval = 3000}) {
|
||||
const [status, setStatus] = useState(null);
|
||||
const [error, setError] = useState(null);
|
||||
const requestedInterval = useRef(pollInterval);
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
try {
|
||||
const data = await fetchFilesProgress();
|
||||
setStatus(data);
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
setError(err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Allow components to request faster polling temporarily
|
||||
const setFastPolling = useCallback((fast) => {
|
||||
requestedInterval.current = fast ? 500 : pollInterval;
|
||||
}, [pollInterval]);
|
||||
|
||||
useEffect(() => {
|
||||
let timeoutId;
|
||||
const poll = async () => {
|
||||
await refresh();
|
||||
timeoutId = setTimeout(poll, requestedInterval.current);
|
||||
};
|
||||
poll();
|
||||
return () => clearTimeout(timeoutId);
|
||||
}, [refresh]);
|
||||
|
||||
return (
|
||||
<FileWorkerStatusContext.Provider value={{status, error, refresh, setFastPolling}}>
|
||||
{children}
|
||||
</FileWorkerStatusContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
export function useFileWorkerStatus() {
|
||||
const context = useContext(FileWorkerStatusContext);
|
||||
if (!context) {
|
||||
throw new Error('useFileWorkerStatus must be used within FileWorkerStatusProvider');
|
||||
}
|
||||
return context;
|
||||
}
|
||||
|
||||
// Convenience hook for reorganization-specific status
|
||||
export function useReorganizationStatus() {
|
||||
const {status: progress, refresh} = useFileWorkerStatus();
|
||||
|
||||
const isReorganizing = progress?.status === 'reorganizing' ||
|
||||
progress?.status === 'batch_reorganizing';
|
||||
|
||||
return {
|
||||
isReorganizing,
|
||||
taskType: progress?.task_type,
|
||||
collectionId: progress?.collection_id,
|
||||
collectionKind: progress?.collection_kind,
|
||||
batchStatus: progress?.batch_status,
|
||||
workerStatus: progress,
|
||||
refresh,
|
||||
};
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@ import {
|
|||
fetchChannels,
|
||||
fetchDecoded,
|
||||
fetchDomains,
|
||||
fetchFilesProgress,
|
||||
filesSearch,
|
||||
getArchive,
|
||||
getChannel,
|
||||
|
|
@ -588,7 +587,7 @@ export const useChannel = (channel_id) => {
|
|||
return {
|
||||
channel: form.formData,
|
||||
form,
|
||||
fetchChannel,
|
||||
fetchChannel: form.fetcher,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -760,34 +759,6 @@ export const useBrowseFiles = () => {
|
|||
return {browseFiles, openFolders, setOpenFolders, fetchFiles};
|
||||
}
|
||||
|
||||
export const useFilesProgress = () => {
|
||||
const [progress, setProgress] = useState(null);
|
||||
|
||||
const localFetchFilesProgress = async () => {
|
||||
try {
|
||||
let p = await fetchFilesProgress();
|
||||
setProgress(p);
|
||||
} catch (e) {
|
||||
setProgress(null);
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
localFetchFilesProgress();
|
||||
}, []);
|
||||
|
||||
return {progress, fetchFilesProgress: localFetchFilesProgress}
|
||||
}
|
||||
|
||||
export const useFilesProgressInterval = () => {
|
||||
const {progress, fetchFilesProgress} = useFilesProgress();
|
||||
|
||||
useRecurringTimeout(fetchFilesProgress, 1000);
|
||||
|
||||
return {progress, fetchFilesProgress};
|
||||
}
|
||||
|
||||
export const useHotspot = () => {
|
||||
const [on, setOn] = useState(null);
|
||||
const [inUse, setInUse] = useState(false);
|
||||
|
|
|
|||
|
|
@ -3,11 +3,50 @@
|
|||
* Provides custom render functions with necessary context providers
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import React, {createContext, useContext} from 'react';
|
||||
import {render} from '@testing-library/react';
|
||||
import {BrowserRouter} from 'react-router';
|
||||
import {MediaContextProvider, ThemeContext} from './contexts/contexts';
|
||||
|
||||
// Mock FileWorkerStatusContext for tests - avoids API calls during testing
|
||||
const MockFileWorkerStatusContext = createContext(null);
|
||||
|
||||
function MockFileWorkerStatusProvider({children}) {
|
||||
const mockValue = {
|
||||
status: null,
|
||||
error: null,
|
||||
refresh: jest.fn(),
|
||||
setFastPolling: jest.fn(),
|
||||
};
|
||||
return (
|
||||
<MockFileWorkerStatusContext.Provider value={mockValue}>
|
||||
{children}
|
||||
</MockFileWorkerStatusContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
// Re-export hooks that use the mock context
|
||||
export function useFileWorkerStatus() {
|
||||
const context = useContext(MockFileWorkerStatusContext);
|
||||
if (!context) {
|
||||
throw new Error('useFileWorkerStatus must be used within FileWorkerStatusProvider');
|
||||
}
|
||||
return context;
|
||||
}
|
||||
|
||||
export function useReorganizationStatus() {
|
||||
const {status: progress, refresh} = useFileWorkerStatus();
|
||||
return {
|
||||
isReorganizing: false,
|
||||
taskType: null,
|
||||
collectionId: null,
|
||||
collectionKind: null,
|
||||
batchStatus: null,
|
||||
workerStatus: progress,
|
||||
refresh,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Custom render function that wraps components with necessary providers
|
||||
* @param {React.Component} ui - Component to render
|
||||
|
|
@ -40,9 +79,11 @@ export function renderWithProviders(
|
|||
function Wrapper({children}) {
|
||||
const content = (
|
||||
<BrowserRouter>
|
||||
<ThemeContext.Provider value={defaultThemeContext}>
|
||||
{children}
|
||||
</ThemeContext.Provider>
|
||||
<MockFileWorkerStatusProvider>
|
||||
<ThemeContext.Provider value={defaultThemeContext}>
|
||||
{children}
|
||||
</ThemeContext.Provider>
|
||||
</MockFileWorkerStatusProvider>
|
||||
</BrowserRouter>
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from typing import List, Tuple, Iterable
|
|||
from sqlalchemy import not_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM
|
||||
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM, FIREFOX
|
||||
from wrolpi.collections import Collection
|
||||
from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
|
||||
slow_logger, get_title_from_html, TRACE_LEVEL
|
||||
|
|
@ -86,11 +86,29 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
|
||||
async def do_singlefile(self, download: Download) -> bytes:
|
||||
"""Create a Singlefile from the archive's URL."""
|
||||
# Get browser settings from config
|
||||
config = lib.get_archive_downloader_config()
|
||||
|
||||
# Determine browser executable
|
||||
browser_executable = config.browser_executable
|
||||
if browser_executable:
|
||||
# Use configured browser path
|
||||
browser = browser_executable
|
||||
else:
|
||||
# Auto-detect: prefer Chromium, fallback to Firefox
|
||||
browser = CHROMIUM or FIREFOX
|
||||
if not browser:
|
||||
raise RuntimeError('No browser found. Install Chromium or Firefox, or configure browser_executable.')
|
||||
|
||||
# Get browser args and user agent from config
|
||||
browser_args = config.browser_args or '["--no-sandbox"]'
|
||||
user_agent = config.user_agent or DOWNLOAD_USER_AGENT
|
||||
|
||||
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
|
||||
str(SINGLE_FILE_BIN),
|
||||
'--browser-executable-path', CHROMIUM,
|
||||
'--browser-args', '["--no-sandbox"]',
|
||||
'--user-agent', DOWNLOAD_USER_AGENT,
|
||||
'--browser-executable-path', str(browser),
|
||||
'--browser-args', browser_args,
|
||||
'--user-agent', user_agent,
|
||||
'--dump-content',
|
||||
'--load-deferred-images-dispatch-scroll-event',
|
||||
download.url,
|
||||
|
|
@ -233,17 +251,7 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive:
|
|||
|
||||
file_group.title = file_group.a_text = title or archive.file_group.title
|
||||
file_group.d_text = contents
|
||||
# Store paths as filenames only (relative to directory), resolved via file_group.resolve_path()
|
||||
file_group.data = {
|
||||
'id': archive.id,
|
||||
'domain': archive.domain,
|
||||
'readability_json_path': archive.readability_json_path.name if archive.readability_json_path else None,
|
||||
'readability_path': archive.readability_path.name if archive.readability_path else None,
|
||||
'readability_txt_path': archive.readability_txt_path.name if archive.readability_txt_path else None,
|
||||
'screenshot_path': archive.screenshot_path.name if archive.screenshot_path else None,
|
||||
'singlefile_path': archive.singlefile_path.name if archive.singlefile_path else None,
|
||||
'info_json_path': archive.info_json_path.name if archive.info_json_path else None,
|
||||
}
|
||||
archive.rebuild_data()
|
||||
|
||||
return archive
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -7,12 +7,14 @@ from sanic_ext import validate
|
|||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi.api_utils import json_response, api_app
|
||||
from wrolpi.cmd import get_installed_browsers
|
||||
from wrolpi.common import logger, wrol_mode_check, api_param_limiter, TRACE_LEVEL
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.schema import JSONErrorResponse
|
||||
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
|
||||
from wrolpi.vars import DOCKERIZED
|
||||
from . import lib, schema
|
||||
|
||||
NAME = 'archive'
|
||||
|
|
@ -90,6 +92,28 @@ async def post_file_format(_: Request, body: schema.ArchiveFileFormatRequest):
|
|||
), status=HTTPStatus.BAD_REQUEST)
|
||||
|
||||
|
||||
@archive_bp.get('/browsers')
|
||||
@openapi.definition(
|
||||
summary='Get list of installed browsers that can be used for archiving.',
|
||||
description='Returns browsers installed on the system. Only available on native deployments (not Docker).',
|
||||
)
|
||||
async def get_browsers(_: Request):
|
||||
"""Get list of installed browsers for SingleFile."""
|
||||
if DOCKERIZED:
|
||||
# Browser configuration is not available in Docker
|
||||
return json_response(dict(
|
||||
browsers=[],
|
||||
available=False,
|
||||
message='Browser configuration is not available in Docker deployments.',
|
||||
))
|
||||
|
||||
browsers = get_installed_browsers()
|
||||
return json_response(dict(
|
||||
browsers=browsers,
|
||||
available=True,
|
||||
))
|
||||
|
||||
|
||||
@archive_bp.get('/upload')
|
||||
@openapi.definition(
|
||||
summary='A message to confirm to the user that they have the correct upload URL.'
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from wrolpi.cmd import READABILITY_BIN, run_command
|
|||
from wrolpi.collections import Collection
|
||||
from wrolpi.common import get_media_directory, get_relative_to_media_directory, logger, extract_domain, \
|
||||
escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \
|
||||
get_wrolpi_config, html_screenshot, ConfigFile
|
||||
get_wrolpi_config, html_screenshot, ConfigFile, trim_file_name
|
||||
from wrolpi.dates import now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
|
||||
|
|
@ -218,12 +218,26 @@ class ArchiveDownloaderConfigValidator:
|
|||
"""Validator for archives_downloader.yaml config."""
|
||||
file_name_format: str
|
||||
version: int = 0
|
||||
# Browser settings (only used on native deployments, not Docker)
|
||||
browser_executable: str = None # None means auto-detect, or absolute path
|
||||
browser_args: str = '["--no-sandbox"]' # JSON array of browser arguments
|
||||
user_agent: str = None # None means use system default
|
||||
|
||||
def __post_init__(self):
|
||||
# Validate file_name_format ends with .%(ext)s (like video format)
|
||||
if not self.file_name_format.endswith('.%(ext)s'):
|
||||
raise ValueError('file_name_format must end with .%(ext)s')
|
||||
|
||||
# Validate browser_args is valid JSON array
|
||||
if self.browser_args:
|
||||
import json
|
||||
try:
|
||||
args = json.loads(self.browser_args)
|
||||
if not isinstance(args, list):
|
||||
raise ValueError('browser_args must be a JSON array')
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f'browser_args must be valid JSON: {e}')
|
||||
|
||||
|
||||
class ArchiveDownloaderConfig(ConfigFile):
|
||||
"""
|
||||
|
|
@ -235,6 +249,9 @@ class ArchiveDownloaderConfig(ConfigFile):
|
|||
Format:
|
||||
version: 0
|
||||
file_name_format: '%(download_date)s_%(title)s.%(ext)s'
|
||||
browser_executable: null # null for auto-detect, or absolute path
|
||||
browser_args: '["--no-sandbox"]' # JSON array of browser arguments
|
||||
user_agent: null # null for system default
|
||||
|
||||
Variables available in file_name_format:
|
||||
- %(title)s - Page title (extracted from HTML)
|
||||
|
|
@ -248,18 +265,41 @@ class ArchiveDownloaderConfig(ConfigFile):
|
|||
|
||||
Subdirectories can be included in the format, e.g.:
|
||||
'%(download_year)s/%(download_datetime)s_%(title)s.%(ext)s'
|
||||
|
||||
Browser settings (only used on native deployments, not Docker):
|
||||
- browser_executable: Path to browser or null for auto-detect
|
||||
- browser_args: JSON array of arguments passed to browser
|
||||
- user_agent: Custom user agent string or null for default
|
||||
"""
|
||||
file_name = 'archives_downloader.yaml'
|
||||
validator = ArchiveDownloaderConfigValidator
|
||||
default_config = dict(
|
||||
version=0,
|
||||
file_name_format='%(download_datetime)s_%(title)s.%(ext)s',
|
||||
browser_executable=None,
|
||||
browser_args='["--no-sandbox"]',
|
||||
user_agent=None,
|
||||
)
|
||||
|
||||
@property
|
||||
def file_name_format(self) -> str:
|
||||
return self._config['file_name_format']
|
||||
|
||||
@property
|
||||
def browser_executable(self) -> Optional[str]:
|
||||
"""Get the configured browser executable path, or None for auto-detect."""
|
||||
return self._config.get('browser_executable')
|
||||
|
||||
@property
|
||||
def browser_args(self) -> str:
|
||||
"""Get the browser arguments as a JSON string."""
|
||||
return self._config.get('browser_args', '["--no-sandbox"]')
|
||||
|
||||
@property
|
||||
def user_agent(self) -> Optional[str]:
|
||||
"""Get the custom user agent, or None for system default."""
|
||||
return self._config.get('user_agent')
|
||||
|
||||
def import_config(self, file: pathlib.Path = None, send_events=False):
|
||||
super().import_config(file, send_events)
|
||||
self.successful_import = True
|
||||
|
|
@ -314,11 +354,20 @@ def format_archive_filename(
|
|||
)
|
||||
|
||||
try:
|
||||
return template % variables
|
||||
result = template % variables
|
||||
# Trim the filename portion if it's too long (may include subdirectory paths)
|
||||
if '/' in result:
|
||||
# Has subdirectories - only trim the filename part
|
||||
parts = result.rsplit('/', 1)
|
||||
subdir = parts[0]
|
||||
filename = trim_file_name(parts[1])
|
||||
return f'{subdir}/{filename}'
|
||||
else:
|
||||
return trim_file_name(result)
|
||||
except KeyError as e:
|
||||
logger.error(f'Invalid variable in archive file_name_format: {e}')
|
||||
# Fallback to default format
|
||||
return f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html'
|
||||
return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
|
||||
|
||||
|
||||
def preview_archive_filename(file_name_format: str) -> str:
|
||||
|
|
@ -356,6 +405,69 @@ def preview_archive_filename(file_name_format: str) -> str:
|
|||
raise RuntimeError(f'Invalid format: {e}')
|
||||
|
||||
|
||||
def format_archive_filename_from_archive(
|
||||
archive: Archive,
|
||||
file_name_format: str = None,
|
||||
) -> str:
|
||||
"""Single source of truth for archive filenames when reorganizing.
|
||||
|
||||
Formats an archive filename using the configured file_name_format template.
|
||||
Used by collection reorganization to compute new file paths.
|
||||
|
||||
Args:
|
||||
archive: Archive model with metadata
|
||||
file_name_format: Optional override, defaults to config value
|
||||
|
||||
Returns:
|
||||
Formatted filename (may include subdirectory path like "2025/My Article.html")
|
||||
"""
|
||||
config = get_archive_downloader_config()
|
||||
template = file_name_format or config.file_name_format
|
||||
|
||||
file_group = archive.file_group
|
||||
|
||||
# Get download date from file_group
|
||||
download_date = file_group.download_datetime or file_group.published_datetime or now()
|
||||
|
||||
# Get title
|
||||
title = file_group.title or 'untitled'
|
||||
|
||||
# Get domain from URL
|
||||
domain = ''
|
||||
if file_group.url:
|
||||
try:
|
||||
domain = extract_domain(file_group.url)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
variables = dict(
|
||||
title=escape_file_name(title) if title else 'untitled',
|
||||
domain=domain,
|
||||
download_datetime=archive_strftime(download_date),
|
||||
download_date=download_date.strftime('%Y-%m-%d'),
|
||||
download_year=str(download_date.year),
|
||||
download_month=f'{download_date.month:02d}',
|
||||
download_day=f'{download_date.day:02d}',
|
||||
ext='html',
|
||||
)
|
||||
|
||||
try:
|
||||
result = template % variables
|
||||
# Trim the filename portion if it's too long (may include subdirectory paths)
|
||||
if '/' in result:
|
||||
# Has subdirectories - only trim the filename part
|
||||
parts = result.rsplit('/', 1)
|
||||
subdir = parts[0]
|
||||
filename = trim_file_name(parts[1])
|
||||
return f'{subdir}/{filename}'
|
||||
else:
|
||||
return trim_file_name(result)
|
||||
except KeyError as e:
|
||||
logger.error(f'Invalid variable in archive file_name_format: {e}')
|
||||
# Fallback to default format
|
||||
return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArchiveFiles:
|
||||
"""Every Archive will have some of these files."""
|
||||
|
|
@ -416,17 +528,18 @@ def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.P
|
|||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
singlefile_path = full_path
|
||||
stem = full_path.stem
|
||||
readability_path = full_path.parent / f'{stem}.readability.html'
|
||||
readability_txt_path = full_path.parent / f'{stem}.readability.txt'
|
||||
readability_json_path = full_path.parent / f'{stem}.readability.json'
|
||||
screenshot_path = full_path.parent / f'{stem}.png'
|
||||
parent_dir = full_path.parent
|
||||
readability_path = trim_file_name(parent_dir / f'{stem}.readability.html')
|
||||
readability_txt_path = trim_file_name(parent_dir / f'{stem}.readability.txt')
|
||||
readability_json_path = trim_file_name(parent_dir / f'{stem}.readability.json')
|
||||
screenshot_path = trim_file_name(parent_dir / f'{stem}.png')
|
||||
else:
|
||||
singlefile_path = directory / formatted_filename
|
||||
stem = singlefile_path.stem
|
||||
readability_path = directory / f'{stem}.readability.html'
|
||||
readability_txt_path = directory / f'{stem}.readability.txt'
|
||||
readability_json_path = directory / f'{stem}.readability.json'
|
||||
screenshot_path = directory / f'{stem}.png'
|
||||
readability_path = trim_file_name(directory / f'{stem}.readability.html')
|
||||
readability_txt_path = trim_file_name(directory / f'{stem}.readability.txt')
|
||||
readability_json_path = trim_file_name(directory / f'{stem}.readability.json')
|
||||
screenshot_path = trim_file_name(directory / f'{stem}.png')
|
||||
|
||||
paths = (singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path)
|
||||
|
||||
|
|
@ -592,6 +705,15 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
|
|||
archive.file_group.download_datetime = now()
|
||||
archive.url = url
|
||||
archive.collection = get_or_create_domain_collection(session, url)
|
||||
|
||||
# Update collection's file_format to track the format used for this file
|
||||
config = get_archive_downloader_config()
|
||||
if archive.collection and config.file_name_format:
|
||||
if not archive.collection.file_format:
|
||||
# Set initial file_format on collection
|
||||
archive.collection.file_format = config.file_name_format
|
||||
save_domains_config.activate_switch()
|
||||
|
||||
archive.flush()
|
||||
|
||||
return archive
|
||||
|
|
|
|||
|
|
@ -351,6 +351,23 @@ class Archive(Base, ModelHelper):
|
|||
if metadata.modified_datetime:
|
||||
self.file_group.published_modified_datetime = metadata.modified_datetime
|
||||
|
||||
def rebuild_data(self):
|
||||
"""Rebuild the data field with current file paths.
|
||||
|
||||
Called during validate() to ensure data field reflects actual filenames
|
||||
after reorganization or other file operations.
|
||||
"""
|
||||
self.file_group.data = {
|
||||
'id': self.id,
|
||||
'domain': self.domain,
|
||||
'readability_json_path': self.readability_json_path.name if self.readability_json_path else None,
|
||||
'readability_path': self.readability_path.name if self.readability_path else None,
|
||||
'readability_txt_path': self.readability_txt_path.name if self.readability_txt_path else None,
|
||||
'screenshot_path': self.screenshot_path.name if self.screenshot_path else None,
|
||||
'singlefile_path': self.singlefile_path.name if self.singlefile_path else None,
|
||||
'info_json_path': self.info_json_path.name if self.info_json_path else None,
|
||||
}
|
||||
|
||||
def validate(self):
|
||||
"""Fill in any missing data about this Archive from its files."""
|
||||
try:
|
||||
|
|
@ -359,6 +376,7 @@ class Archive(Base, ModelHelper):
|
|||
self.apply_domain()
|
||||
self.apply_singlefile_title()
|
||||
self.apply_metadata()
|
||||
self.rebuild_data()
|
||||
except Exception as e:
|
||||
logger.warning(f'Unable to validate {self}', exc_info=e)
|
||||
if PYTEST:
|
||||
|
|
|
|||
|
|
@ -342,18 +342,9 @@ async def test_archive_generate_screenshot(test_session, async_client, archive_f
|
|||
assert original_screenshot_path is not None, 'Archive should have a screenshot'
|
||||
assert original_screenshot_path.is_file(), 'Screenshot file should exist'
|
||||
|
||||
# Simulate the bug: archive has screenshot file but data['screenshot_path'] is not set
|
||||
# (This could happen if the archive was created before set_screenshot was implemented)
|
||||
if archive.file_group.data and 'screenshot_path' in archive.file_group.data:
|
||||
data = dict(archive.file_group.data)
|
||||
del data['screenshot_path']
|
||||
archive.file_group.data = data
|
||||
test_session.commit()
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
# Verify data was cleared but file still exists
|
||||
assert 'screenshot_path' not in (archive.file_group.data or {}), 'screenshot_path should not be in data yet'
|
||||
assert archive.screenshot_path is not None, 'But screenshot file should still exist'
|
||||
# Note: Previously this test simulated a bug where data['screenshot_path'] could be missing
|
||||
# while the file existed. This scenario is now auto-fixed by Archive.validate() which calls
|
||||
# rebuild_data(), so the simulation is no longer necessary.
|
||||
|
||||
request, response = await async_client.post(f'/api/archive/{archive.id}/generate_screenshot')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
|
@ -364,11 +355,11 @@ async def test_archive_generate_screenshot(test_session, async_client, archive_f
|
|||
assert archive.screenshot_path == original_screenshot_path, 'Screenshot path should be unchanged'
|
||||
assert archive.screenshot_path.is_file(), 'Screenshot file should still exist'
|
||||
assert archive.screenshot_file is not None, 'Screenshot file should still be tracked'
|
||||
# IMPORTANT: Also verify FileGroup.data was updated (this catches the bug where only files was updated)
|
||||
# IMPORTANT: Also verify FileGroup.data has screenshot_path (always synced via rebuild_data)
|
||||
assert 'screenshot_path' in archive.file_group.data, \
|
||||
'screenshot_path should be in FileGroup.data after ensuring tracking'
|
||||
assert str(archive.file_group.data['screenshot_path']) == str(original_screenshot_path), \
|
||||
'screenshot_path in data should match the file path'
|
||||
'screenshot_path should be in FileGroup.data'
|
||||
assert archive.file_group.data['screenshot_path'] == original_screenshot_path.name, \
|
||||
'screenshot_path in data should be filename only'
|
||||
|
||||
# Test error case: Archive has no singlefile
|
||||
archive_no_singlefile = archive_factory('example.com', 'https://example.com/no-singlefile', screenshot=False)
|
||||
|
|
@ -428,3 +419,28 @@ async def test_archive_file_format_preview(async_client):
|
|||
request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert 'error' in response.json
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_browsers_endpoint(async_client):
|
||||
"""Test the browsers endpoint returns installed browsers."""
|
||||
# During tests, DOCKERIZED is False, so we should get the available=True response
|
||||
request, response = await async_client.get('/api/archive/browsers')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert 'browsers' in response.json
|
||||
assert 'available' in response.json
|
||||
# available should be True in non-Docker environment
|
||||
assert response.json['available'] is True
|
||||
# browsers is a list
|
||||
assert isinstance(response.json['browsers'], list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_browsers_endpoint_dockerized(async_client):
|
||||
"""Test the browsers endpoint returns not available when dockerized."""
|
||||
with patch('modules.archive.api.DOCKERIZED', True):
|
||||
request, response = await async_client.get('/api/archive/browsers')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['browsers'] == []
|
||||
assert response.json['available'] is False
|
||||
assert 'message' in response.json
|
||||
|
|
|
|||
|
|
@ -365,14 +365,17 @@ async def test_get_title_from_html(test_directory, test_session, fake_now, async
|
|||
|
||||
@skip_circleci
|
||||
def test_get_new_archive_files_length(test_directory, fake_now):
|
||||
"""Archive titles are truncated to fit file system length limit. (255-character limit for most file systems)"""
|
||||
"""Archive titles are truncated to fit file system BYTE limit (200 bytes - see MAXIMUM_FILE_BYTES)."""
|
||||
from wrolpi.common import MAXIMUM_FILE_BYTES
|
||||
|
||||
fake_now(datetime(2001, 1, 1))
|
||||
archive_files = get_new_archive_files('https://example.com', 'a' * 500)
|
||||
assert len(str(archive_files.singlefile.name)) == 225
|
||||
assert len(str(archive_files.readability.name)) == 237
|
||||
assert len(str(archive_files.readability_txt.name)) == 236
|
||||
assert len(str(archive_files.readability_json.name)) == 237
|
||||
assert len(str(archive_files.screenshot.name)) == 224
|
||||
# All filenames should be <= MAXIMUM_FILE_BYTES (bytes, not characters)
|
||||
assert len(str(archive_files.singlefile.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
|
||||
assert len(str(archive_files.readability.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
|
||||
assert len(str(archive_files.readability_txt.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
|
||||
assert len(str(archive_files.readability_json.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
|
||||
assert len(str(archive_files.screenshot.name).encode('utf-8')) <= MAXIMUM_FILE_BYTES
|
||||
|
||||
|
||||
@skip_circleci
|
||||
|
|
@ -1289,6 +1292,26 @@ def test_archive_downloader_config_validation():
|
|||
with pytest.raises(ValueError, match='must end with'):
|
||||
ArchiveDownloaderConfigValidator(file_name_format='%(download_date)s_%(title)s')
|
||||
|
||||
# Valid browser_args (JSON array)
|
||||
ArchiveDownloaderConfigValidator(
|
||||
file_name_format='%(title)s.%(ext)s',
|
||||
browser_args='["--no-sandbox", "--headless"]'
|
||||
)
|
||||
|
||||
# Invalid browser_args (not valid JSON)
|
||||
with pytest.raises(ValueError, match='must be valid JSON'):
|
||||
ArchiveDownloaderConfigValidator(
|
||||
file_name_format='%(title)s.%(ext)s',
|
||||
browser_args='not valid json'
|
||||
)
|
||||
|
||||
# Invalid browser_args (not an array)
|
||||
with pytest.raises(ValueError, match='must be a JSON array'):
|
||||
ArchiveDownloaderConfigValidator(
|
||||
file_name_format='%(title)s.%(ext)s',
|
||||
browser_args='{"key": "value"}'
|
||||
)
|
||||
|
||||
|
||||
def test_format_archive_filename_default(test_directory, fake_now):
|
||||
"""format_archive_filename uses default format."""
|
||||
|
|
|
|||
|
|
@ -128,3 +128,53 @@ async def test_set_screenshot(async_client, test_session, archive_factory, image
|
|||
|
||||
# Verify no screenshot was set
|
||||
assert archive_no_screenshot.screenshot_path is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_validate_rebuilds_data_after_filename_change(async_client, test_session, test_directory,
|
||||
archive_factory):
|
||||
"""Archive.validate() should rebuild data field when filenames change.
|
||||
|
||||
This tests the bug where reorganization updates files[] but not data{}.
|
||||
The data field contains cached path names that must stay in sync.
|
||||
|
||||
The bug scenario: files are renamed during reorganization (e.g., fixing separator
|
||||
from '___' to '_'), files[] is updated but data{} is not. The UI then uses
|
||||
stale data.singlefile_path and gets 404 errors.
|
||||
"""
|
||||
# Create an archive using the factory
|
||||
archive = archive_factory(domain='test.com', url='https://test.com/article')
|
||||
test_session.commit()
|
||||
|
||||
# Verify initial data field has singlefile_path
|
||||
assert archive.file_group.data is not None
|
||||
original_singlefile = archive.file_group.data.get('singlefile_path')
|
||||
assert original_singlefile is not None
|
||||
|
||||
# Simulate what reorganization does: update files[] but not data{}
|
||||
# (This mimics _bulk_update_file_groups_reorganize behavior)
|
||||
# Use a realistic rename: change the title part of the filename
|
||||
# Original: '2000-01-01-07-00-01_NA.html' -> '2000-01-01-07-00-01_New-Title.html'
|
||||
new_filename = original_singlefile.replace('_NA.html', '_New-Title.html')
|
||||
old_files = archive.file_group.files
|
||||
new_files = []
|
||||
for f in old_files:
|
||||
new_f = dict(f)
|
||||
if f['path'].endswith('.html') and not f['path'].endswith('.readability.html'):
|
||||
new_f['path'] = new_filename
|
||||
new_files.append(new_f)
|
||||
archive.file_group.files = new_files
|
||||
archive.file_group.primary_path = archive.file_group.directory / new_filename
|
||||
|
||||
# Rename the actual file on disk to match the new filename
|
||||
old_path = archive.file_group.directory / original_singlefile
|
||||
new_path = archive.file_group.directory / new_filename
|
||||
old_path.rename(new_path)
|
||||
|
||||
# data field still has OLD filename (this is the bug)
|
||||
assert archive.file_group.data['singlefile_path'] == original_singlefile
|
||||
|
||||
# After validate(), data field should have NEW filename
|
||||
archive.validate()
|
||||
|
||||
assert archive.file_group.data['singlefile_path'] == new_filename
|
||||
|
|
|
|||
|
|
@ -260,6 +260,9 @@ async def test_channel_post_directory(test_session, async_client, test_directory
|
|||
async def test_channel_by_id(test_session, async_client, simple_channel, simple_video):
|
||||
request, response = await async_client.get(f'/api/videos/channels/{simple_channel.id}')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
channel = response.json['channel']
|
||||
# needs_reorganization should be included for UI to display reorganization status
|
||||
assert 'needs_reorganization' in channel
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -139,7 +139,10 @@ def convert_wrolpi_filename_format(filename_format: str) -> str:
|
|||
|
||||
|
||||
def preview_filename(filename_format: str) -> str:
|
||||
"""Return an example video file name formatted using the provided filename format.
|
||||
"""Return an example video file name using simple string formatting.
|
||||
|
||||
This is a lightweight implementation that doesn't require YoutubeDL instantiation,
|
||||
which is expensive (~1500 extractors loaded, regexes compiled).
|
||||
|
||||
@raise RuntimeError: If the format is invalid.
|
||||
"""
|
||||
|
|
@ -149,21 +152,31 @@ def preview_filename(filename_format: str) -> str:
|
|||
# Convert WROLPi-specific variables to yt-dlp syntax
|
||||
converted_format = convert_wrolpi_filename_format(filename_format)
|
||||
|
||||
options = get_videos_downloader_config().yt_dlp_options
|
||||
options['outtmpl'] = converted_format
|
||||
ydl = YoutubeDL(copy.deepcopy(options))
|
||||
entry = dict(
|
||||
sample_date = now()
|
||||
variables = dict(
|
||||
uploader='WROLPi',
|
||||
timestamp=int(now().timestamp()),
|
||||
upload_date=now().strftime('%Y%m%d'),
|
||||
channel='WROLPi',
|
||||
upload_date=sample_date.strftime('%Y%m%d'),
|
||||
id='Qz-FuenRylQ',
|
||||
title='The title of the video',
|
||||
ext='mp4',
|
||||
description='A description of the video',
|
||||
timestamp=int(sample_date.timestamp()),
|
||||
)
|
||||
filename = prepare_filename(entry, ydl=ydl).lstrip('/')
|
||||
|
||||
return filename
|
||||
# Handle yt-dlp's date extraction syntax %(upload_date>%Y)s and %(upload_date>%m)s
|
||||
# Replace these with their values before standard Python % formatting
|
||||
def replace_date_format(match):
|
||||
date_format = match.group(1)
|
||||
return sample_date.strftime(date_format)
|
||||
|
||||
converted_format = re.sub(r'%\(upload_date>([^)]+)\)s', replace_date_format, converted_format)
|
||||
|
||||
try:
|
||||
filename = converted_format % variables
|
||||
return filename.lstrip('/')
|
||||
except KeyError as e:
|
||||
raise RuntimeError(f'Invalid variable: {e}')
|
||||
|
||||
|
||||
@cached_multiprocessing_result
|
||||
|
|
@ -545,6 +558,17 @@ class VideoDownloader(Downloader, ABC):
|
|||
video_id, video_info_json_path = video.id, video.info_json_path
|
||||
if video_info_json_path and (new_info_json := video.clean_info_json()):
|
||||
video.replace_info_json(new_info_json, clean=False)
|
||||
|
||||
# Update channel's collection's file_format to track the format used
|
||||
if channel_id:
|
||||
from modules.videos.models import Channel
|
||||
from modules.videos.lib import save_channels_config
|
||||
channel_obj = session.query(Channel).get(channel_id)
|
||||
if channel_obj and channel_obj.collection and not channel_obj.collection.file_format:
|
||||
video_config = get_videos_downloader_config()
|
||||
channel_obj.collection.file_format = video_config.file_name_format
|
||||
save_channels_config.activate_switch()
|
||||
|
||||
session.commit()
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from modules.videos.models import Video
|
|||
from wrolpi import dates, flags
|
||||
from wrolpi.captions import extract_captions
|
||||
from wrolpi.common import ConfigFile, extract_domain, logger, \
|
||||
escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config
|
||||
escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config, trim_file_name
|
||||
from wrolpi.dates import Seconds, from_timestamp
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
|
|
@ -948,3 +948,85 @@ def browser_profile_to_yt_dlp_arg(profile: pathlib.Path) -> str:
|
|||
"""Takes a Path and returns a string that can be used as a yt-dlp `--cookies-from-browser` argument."""
|
||||
*_, browser, profile = str(profile).split('/')
|
||||
return f'{browser}:{profile}'
|
||||
|
||||
|
||||
def format_video_filename(
|
||||
video: Video,
|
||||
file_name_format: str = None,
|
||||
) -> str:
|
||||
"""Single source of truth for video filenames.
|
||||
|
||||
Formats a video filename using the configured file_name_format template.
|
||||
Used by both the video downloader (initial download) and collection reorganization.
|
||||
|
||||
Args:
|
||||
video: Video model with metadata (upload_date, uploader, title, source_id)
|
||||
file_name_format: Optional override, defaults to config value
|
||||
|
||||
Returns:
|
||||
Formatted filename (may include subdirectory path like "2025/01/video.mp4")
|
||||
|
||||
Example:
|
||||
With format '%(upload_year)s/%(upload_month)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s':
|
||||
Returns '2025/01/WROLPi_20250115_abc123_My Video Title.mp4'
|
||||
"""
|
||||
from modules.videos.downloader import convert_wrolpi_filename_format
|
||||
|
||||
config = get_videos_downloader_config()
|
||||
template = file_name_format or config.file_name_format
|
||||
|
||||
# Get metadata from video
|
||||
file_group = video.file_group
|
||||
info_json = video.get_info_json() or {}
|
||||
|
||||
# Extract upload date info
|
||||
upload_date = file_group.published_datetime
|
||||
if not upload_date and info_json.get('upload_date'):
|
||||
upload_date = dates.strpdate(info_json['upload_date'])
|
||||
|
||||
# Build variables dict matching yt-dlp's expected variables
|
||||
# Use info_json if available, fall back to derived values
|
||||
uploader = info_json.get('uploader') or info_json.get('channel') or ''
|
||||
title = file_group.title or info_json.get('title') or info_json.get('fulltitle') or 'untitled'
|
||||
source_id = video.source_id or info_json.get('id') or ''
|
||||
|
||||
# Get video file extension
|
||||
video_path = video.video_path
|
||||
ext = video_path.suffix.lstrip('.') if video_path else config.merge_output_format or 'mp4'
|
||||
|
||||
variables = dict(
|
||||
uploader=escape_file_name(uploader) if uploader else '',
|
||||
channel=escape_file_name(uploader) if uploader else '', # Alias
|
||||
upload_date=upload_date.strftime('%Y%m%d') if upload_date else '',
|
||||
id=source_id,
|
||||
title=escape_file_name(title) if title else 'untitled',
|
||||
ext=ext,
|
||||
# WROLPi-specific variables for subdirectory organization
|
||||
upload_year=str(upload_date.year) if upload_date else '',
|
||||
upload_month=f'{upload_date.month:02d}' if upload_date else '',
|
||||
)
|
||||
|
||||
# Convert WROLPi-specific variables to yt-dlp syntax then apply
|
||||
# (This handles %(upload_year)s -> %(upload_date>%Y)s conversion)
|
||||
converted_template = convert_wrolpi_filename_format(template)
|
||||
|
||||
# For the converted variables (like %(upload_date>%Y)s), we need to add them too
|
||||
variables['upload_date>%Y'] = str(upload_date.year) if upload_date else ''
|
||||
variables['upload_date>%m'] = f'{upload_date.month:02d}' if upload_date else ''
|
||||
|
||||
try:
|
||||
result = converted_template % variables
|
||||
# Trim the filename portion if it's too long (may include subdirectory paths)
|
||||
if '/' in result:
|
||||
# Has subdirectories - only trim the filename part
|
||||
parts = result.rsplit('/', 1)
|
||||
subdir = parts[0]
|
||||
filename = trim_file_name(parts[1])
|
||||
return f'{subdir}/{filename}'
|
||||
else:
|
||||
return trim_file_name(result)
|
||||
except KeyError as e:
|
||||
logger.error(f'Invalid variable in video file_name_format: {e}')
|
||||
# Fallback to a simple format
|
||||
fallback_title = escape_file_name(title) if title else 'untitled'
|
||||
return trim_file_name(f'{fallback_title}.{ext}')
|
||||
|
|
|
|||
|
|
@ -785,6 +785,9 @@ class Channel(ModelHelper, Base):
|
|||
tag_name=self.tag_name,
|
||||
url=self.url or None,
|
||||
)
|
||||
# Include file_format from the underlying Collection
|
||||
if self.collection and self.collection.file_format:
|
||||
config['file_format'] = self.collection.file_format
|
||||
return config
|
||||
|
||||
@classmethod
|
||||
|
|
@ -860,6 +863,7 @@ class Channel(ModelHelper, Base):
|
|||
d['rss_url'] = self.get_rss_url()
|
||||
d['directory'] = \
|
||||
self.directory.relative_to(get_media_directory()) if self.directory else None
|
||||
d['needs_reorganization'] = self.collection.needs_reorganization if self.collection else False
|
||||
if with_statistics:
|
||||
d['statistics'] = self.get_statistics()
|
||||
if with_downloads:
|
||||
|
|
|
|||
|
|
@ -280,3 +280,45 @@ class TestChannelsConfigEdgeCases:
|
|||
# Directory should be relative, not absolute
|
||||
assert not config['directory'].startswith('/'), \
|
||||
f"Expected relative path but got absolute: {config['directory']}"
|
||||
|
||||
async def test_channel_config_view_includes_file_format(self, test_session, test_directory, async_client,
|
||||
test_channels_config, channel_factory):
|
||||
"""Channel.config_view() should include file_format from underlying Collection."""
|
||||
channel = channel_factory()
|
||||
channel.collection.file_format = '%(uploader)s_%(id)s_%(title)s.%(ext)s'
|
||||
test_session.commit()
|
||||
|
||||
config = channel.config_view()
|
||||
assert config['file_format'] == '%(uploader)s_%(id)s_%(title)s.%(ext)s'
|
||||
|
||||
async def test_channel_file_format_round_trip(self, test_session, test_directory, async_client,
|
||||
test_channels_config, channel_factory):
|
||||
"""File format should survive config dump and import cycle."""
|
||||
config = get_channels_config()
|
||||
config_path = config.get_file()
|
||||
|
||||
# Create channel with file_format
|
||||
channel = channel_factory(name='Format Test')
|
||||
channel.collection.file_format = '%(upload_date)s_%(title)s.%(ext)s'
|
||||
test_session.commit()
|
||||
|
||||
# Dump to config
|
||||
config.dump_config()
|
||||
|
||||
# Verify file_format in YAML
|
||||
with open(config_path) as f:
|
||||
config_data = yaml.safe_load(f)
|
||||
channel_config = next(c for c in config_data['channels'] if c['name'] == 'Format Test')
|
||||
assert channel_config['file_format'] == '%(upload_date)s_%(title)s.%(ext)s'
|
||||
|
||||
# Clear the file_format to simulate factory reset
|
||||
channel.collection.file_format = None
|
||||
test_session.commit()
|
||||
|
||||
# Re-import
|
||||
config.initialize()
|
||||
config.import_config()
|
||||
|
||||
# Verify file_format restored
|
||||
test_session.refresh(channel)
|
||||
assert channel.collection.file_format == '%(upload_date)s_%(title)s.%(ext)s'
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ from unittest import mock
|
|||
import pytest
|
||||
|
||||
from modules.videos.downloader import VideoDownloader, \
|
||||
get_or_create_channel, channel_downloader, video_downloader, preview_filename
|
||||
get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
|
||||
prepare_filename, convert_wrolpi_filename_format
|
||||
from modules.videos.lib import get_videos_downloader_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.conftest import test_directory, await_switches
|
||||
|
|
@ -573,6 +574,54 @@ def test_preview_filename_with_upload_year(test_directory, fake_now):
|
|||
== '2000/WROLPi_20000101_The title of the video.mp4'
|
||||
|
||||
|
||||
def test_preview_filename_matches_youtubedl(test_directory, fake_now):
|
||||
"""Ensure our lightweight preview_filename() matches YoutubeDL's actual output.
|
||||
|
||||
This test validates that our simple string formatting produces identical
|
||||
results to YoutubeDL's prepare_filename(). If yt-dlp changes behavior,
|
||||
this test will catch it.
|
||||
"""
|
||||
from copy import deepcopy
|
||||
from yt_dlp import YoutubeDL
|
||||
from wrolpi.dates import now
|
||||
|
||||
test_formats = [
|
||||
'%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
|
||||
'%(upload_date)s_%(id)s_%(title)s.%(ext)s',
|
||||
'%(upload_year)s/%(upload_month)s/%(title)s.%(ext)s',
|
||||
'%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s',
|
||||
'%(channel)s/%(id)s.%(ext)s',
|
||||
]
|
||||
|
||||
# Sample entry used by both methods - must match preview_filename's variables
|
||||
sample_date = now()
|
||||
entry = dict(
|
||||
uploader='WROLPi',
|
||||
channel='WROLPi',
|
||||
timestamp=int(sample_date.timestamp()),
|
||||
upload_date=sample_date.strftime('%Y%m%d'),
|
||||
id='Qz-FuenRylQ',
|
||||
title='The title of the video',
|
||||
ext='mp4',
|
||||
description='A description of the video',
|
||||
)
|
||||
|
||||
for fmt in test_formats:
|
||||
# Get result from our lightweight implementation
|
||||
lightweight_result = preview_filename(fmt)
|
||||
|
||||
# Get result from actual YoutubeDL
|
||||
converted_format = convert_wrolpi_filename_format(fmt)
|
||||
options = get_videos_downloader_config().yt_dlp_options
|
||||
options = deepcopy(options)
|
||||
options['outtmpl'] = converted_format
|
||||
ydl = YoutubeDL(options)
|
||||
ytdlp_result = prepare_filename(entry, ydl=ydl).lstrip('/')
|
||||
|
||||
assert lightweight_result == ytdlp_result, \
|
||||
f"Mismatch for format '{fmt}':\n lightweight: {lightweight_result}\n ytdlp: {ytdlp_result}"
|
||||
|
||||
|
||||
def test_normalize_video_file_names(test_directory, video_download_manager):
|
||||
"""VideoDownloader renames caption files that do not share the video file's stem."""
|
||||
downloader = VideoDownloader()
|
||||
|
|
|
|||
|
|
@ -326,3 +326,32 @@ def test_video_location(test_session, video_factory, channel_factory, async_clie
|
|||
channel = channel_factory(name='ChannelName')
|
||||
video2 = video_factory(title='vid2', channel_id=channel.id)
|
||||
assert video2.location == '/videos/channel/1/video/2'
|
||||
|
||||
|
||||
def test_format_video_filename_without_upload_date(test_session, video_factory, test_videos_downloader_config, caplog):
|
||||
"""Videos without upload_date should not cause KeyError when format uses upload_year."""
|
||||
import logging
|
||||
from modules.videos.lib import format_video_filename
|
||||
|
||||
# Create video without upload_date
|
||||
video = video_factory(title='Test Video', with_video_file=True, with_info_json={'id': 'abc123'})
|
||||
assert video.file_group.published_datetime is None
|
||||
|
||||
# Use a format that includes upload_year subdirectory
|
||||
template = '%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
|
||||
|
||||
# Should not raise KeyError and should not log an error
|
||||
with caplog.at_level(logging.ERROR):
|
||||
result = format_video_filename(video, template)
|
||||
|
||||
# Should not have logged any errors (fallback was not used)
|
||||
assert 'Invalid variable in video file_name_format' not in caplog.text
|
||||
|
||||
# Should return a properly formatted filename with the template structure
|
||||
# (empty year and date components produce a path with subdirectory)
|
||||
assert result.endswith('.mp4')
|
||||
assert 'Test Video' in result
|
||||
# The result should follow the template format, not the fallback format
|
||||
# Template result: '{upload_year}/{uploader}_{upload_date}_{id}_{title}.mp4'
|
||||
# With empty year/date: '/__Test Video_Test Video.mp4' (has a slash for subdirectory)
|
||||
assert '/' in result # Confirms template was used (has subdirectory), not fallback
|
||||
|
|
|
|||
|
|
@ -5,5 +5,5 @@ testpaths =
|
|||
modules
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
# Do not remove timeout - tests should not run forever
|
||||
timeout = 20
|
||||
timeout = 30
|
||||
timeout_method = signal
|
||||
|
|
|
|||
|
|
@ -30,4 +30,4 @@ srt==3.5.3
|
|||
tqdm==4.67.1
|
||||
vininfo[cli]==1.8.0
|
||||
webvtt-py==0.5.1
|
||||
yt-dlp==2025.12.08
|
||||
yt-dlp==2026.01.29
|
||||
|
|
|
|||
|
|
@ -79,11 +79,54 @@ CHROMIUM = which('chromium-browser', 'chromium',
|
|||
'/usr/bin/chromium-browser', # rpi os
|
||||
'/usr/bin/chromium', # debian
|
||||
)
|
||||
FIREFOX = which('firefox', 'firefox-esr',
|
||||
'/usr/bin/firefox',
|
||||
'/usr/bin/firefox-esr',
|
||||
)
|
||||
READABILITY_BIN = which('readability-extractor',
|
||||
'/usr/bin/readability-extractor', # rpi os
|
||||
'/usr/local/bin/readability-extractor', # debian
|
||||
)
|
||||
|
||||
# Known browser definitions for SingleFile
|
||||
KNOWN_BROWSERS = {
|
||||
'chromium': {
|
||||
'paths': ['chromium-browser', 'chromium', '/usr/bin/chromium-browser', '/usr/bin/chromium'],
|
||||
'name': 'Chromium',
|
||||
},
|
||||
'firefox': {
|
||||
'paths': ['firefox', 'firefox-esr', '/usr/bin/firefox', '/usr/bin/firefox-esr'],
|
||||
'name': 'Firefox',
|
||||
},
|
||||
'brave': {
|
||||
'paths': ['brave-browser', '/usr/bin/brave-browser'],
|
||||
'name': 'Brave',
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def get_installed_browsers() -> list:
|
||||
"""
|
||||
Detect browsers installed on the system that can be used with SingleFile.
|
||||
|
||||
Returns a list of dicts with:
|
||||
- key: The browser identifier (e.g., 'chromium', 'firefox')
|
||||
- name: Human-readable name (e.g., 'Chromium', 'Firefox')
|
||||
- path: Absolute path to the executable
|
||||
"""
|
||||
browsers = []
|
||||
|
||||
for key, info in KNOWN_BROWSERS.items():
|
||||
path = which(*info['paths'])
|
||||
if path:
|
||||
browsers.append({
|
||||
'key': key,
|
||||
'name': info['name'],
|
||||
'path': str(path),
|
||||
})
|
||||
|
||||
return browsers
|
||||
|
||||
# Videos
|
||||
YT_DLP_BIN = which(
|
||||
'/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location
|
||||
|
|
|
|||
|
|
@ -266,3 +266,224 @@ async def search_collections_endpoint(request: Request, body: schema.CollectionS
|
|||
'collections': collections,
|
||||
'totals': {'collections': len(collections)}
|
||||
})
|
||||
|
||||
|
||||
@collection_bp.get('/<collection_id:int>/reorganize/preview')
|
||||
@openapi.summary('Preview collection reorganization')
|
||||
@openapi.response(HTTPStatus.OK, schema.ReorganizationPreviewResponse, description="Reorganization preview")
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
async def preview_reorganization_endpoint(request: Request, collection_id: int):
|
||||
"""
|
||||
Get preview with sample of files to be renamed.
|
||||
|
||||
Returns a preview of what would happen if the collection is reorganized.
|
||||
This is useful for showing users what files would move before they commit.
|
||||
|
||||
Returns:
|
||||
- collection_id: The collection ID
|
||||
- collection_name: The collection name
|
||||
- total_files: Total FileGroups in collection
|
||||
- files_needing_move: Count needing reorganization
|
||||
- sample_moves: ~10 example moves [{old_path, new_path}, ...]
|
||||
- new_file_format: The format template that will be applied
|
||||
- current_file_format: The format currently stored on collection
|
||||
"""
|
||||
from wrolpi.collections.reorganize import get_reorganization_preview
|
||||
|
||||
session = request.ctx.session
|
||||
try:
|
||||
# Use exact_count=True for individual collection preview to get accurate count
|
||||
preview = get_reorganization_preview(collection_id, session, exact_count=True)
|
||||
return json_response({
|
||||
'collection_id': preview.collection_id,
|
||||
'collection_name': preview.collection_name,
|
||||
'total_files': preview.total_files,
|
||||
'files_needing_move': preview.files_needing_move,
|
||||
'sample_moves': preview.sample_moves,
|
||||
'new_file_format': preview.new_file_format,
|
||||
'current_file_format': preview.current_file_format,
|
||||
})
|
||||
except UnknownCollection as e:
|
||||
return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND)
|
||||
except ValueError as e:
|
||||
return json_response({'error': str(e)}, status=HTTPStatus.BAD_REQUEST)
|
||||
|
||||
|
||||
@collection_bp.post('/<collection_id:int>/reorganize')
|
||||
@openapi.summary('Execute collection reorganization')
|
||||
@openapi.response(HTTPStatus.OK, schema.ReorganizationExecuteResponse, description="Reorganization started")
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
@wrol_mode_check
|
||||
async def execute_reorganization_endpoint(request: Request, collection_id: int):
|
||||
"""
|
||||
Execute reorganization.
|
||||
|
||||
Moves files to match the current file_name_format configuration.
|
||||
The operation runs in the background via FileWorker.
|
||||
|
||||
Returns:
|
||||
- job_id: For tracking progress via the status endpoint
|
||||
- message: Status message
|
||||
"""
|
||||
from wrolpi.collections.reorganize import execute_reorganization
|
||||
|
||||
session = request.ctx.session
|
||||
try:
|
||||
job_id = execute_reorganization(collection_id, session)
|
||||
if job_id:
|
||||
return json_response({
|
||||
'job_id': job_id,
|
||||
'message': 'Reorganization started'
|
||||
})
|
||||
else:
|
||||
return json_response({
|
||||
'job_id': '',
|
||||
'message': 'No files need reorganization'
|
||||
})
|
||||
except UnknownCollection as e:
|
||||
return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND)
|
||||
except ValueError as e:
|
||||
return json_response({'error': str(e)}, status=HTTPStatus.BAD_REQUEST)
|
||||
|
||||
|
||||
@collection_bp.get('/<collection_id:int>/reorganize/status/<job_id:str>')
|
||||
@openapi.summary('Get reorganization status')
|
||||
@openapi.response(HTTPStatus.OK, schema.ReorganizationStatusResponse, description="Reorganization status")
|
||||
async def get_reorganization_status_endpoint(_: Request, collection_id: int, job_id: str):
|
||||
"""
|
||||
Get status of in-flight reorganization.
|
||||
|
||||
Returns:
|
||||
- status: 'pending' | 'running' | 'complete' | 'failed'
|
||||
- total: Total files to move
|
||||
- completed: Files moved so far
|
||||
- percent: Completion percentage
|
||||
- error: Error message if failed
|
||||
"""
|
||||
from wrolpi.collections.reorganize import get_reorganization_status
|
||||
|
||||
status = get_reorganization_status(job_id)
|
||||
return json_response(status)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Batch Reorganization Endpoints
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@collection_bp.get('/reorganize/channels')
|
||||
@openapi.summary('List channels needing batch reorganization')
|
||||
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationListResponse,
|
||||
description="List of channels needing reorganization")
|
||||
async def list_channels_needing_reorganization_endpoint(request: Request):
|
||||
"""
|
||||
List all channels that need reorganization because their file_name_format
|
||||
differs from the current config.
|
||||
|
||||
Returns:
|
||||
- collections: List of channel info with sample moves
|
||||
- total_collections: Total count needing reorganization
|
||||
- total_files_needing_move: Sum of all files needing reorganization
|
||||
- new_file_format: The config format that will be applied
|
||||
"""
|
||||
from wrolpi.collections.reorganize import get_collections_needing_reorganization
|
||||
|
||||
session = request.ctx.session
|
||||
result = get_collections_needing_reorganization('channel', session)
|
||||
return json_response(result)
|
||||
|
||||
|
||||
@collection_bp.get('/reorganize/domains')
|
||||
@openapi.summary('List domains needing batch reorganization')
|
||||
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationListResponse,
|
||||
description="List of domains needing reorganization")
|
||||
async def list_domains_needing_reorganization_endpoint(request: Request):
|
||||
"""
|
||||
List all domains that need reorganization because their file_name_format
|
||||
differs from the current config.
|
||||
|
||||
Returns:
|
||||
- collections: List of domain info with sample moves
|
||||
- total_collections: Total count needing reorganization
|
||||
- total_files_needing_move: Sum of all files needing reorganization
|
||||
- new_file_format: The config format that will be applied
|
||||
"""
|
||||
from wrolpi.collections.reorganize import get_collections_needing_reorganization
|
||||
|
||||
session = request.ctx.session
|
||||
result = get_collections_needing_reorganization('domain', session)
|
||||
return json_response(result)
|
||||
|
||||
|
||||
@collection_bp.post('/reorganize/channels')
|
||||
@openapi.summary('Execute batch reorganization of all channels')
|
||||
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationExecuteResponse,
|
||||
description="Batch reorganization started")
|
||||
@wrol_mode_check
|
||||
async def execute_batch_reorganization_channels_endpoint(request: Request):
|
||||
"""
|
||||
Execute batch reorganization of all channels that need it.
|
||||
|
||||
Processes channels sequentially. If any channel fails, the batch stops
|
||||
and reports which channel failed.
|
||||
|
||||
Returns:
|
||||
- batch_job_id: For tracking progress via the batch status endpoint
|
||||
- message: Status message
|
||||
- collection_count: Number of channels to process
|
||||
"""
|
||||
from wrolpi.collections.reorganize import execute_batch_reorganization
|
||||
|
||||
session = request.ctx.session
|
||||
result = execute_batch_reorganization('channel', session)
|
||||
return json_response(result)
|
||||
|
||||
|
||||
@collection_bp.post('/reorganize/domains')
|
||||
@openapi.summary('Execute batch reorganization of all domains')
|
||||
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationExecuteResponse,
|
||||
description="Batch reorganization started")
|
||||
@wrol_mode_check
|
||||
async def execute_batch_reorganization_domains_endpoint(request: Request):
|
||||
"""
|
||||
Execute batch reorganization of all domains that need it.
|
||||
|
||||
Processes domains sequentially. If any domain fails, the batch stops
|
||||
and reports which domain failed.
|
||||
|
||||
Returns:
|
||||
- batch_job_id: For tracking progress via the batch status endpoint
|
||||
- message: Status message
|
||||
- collection_count: Number of domains to process
|
||||
"""
|
||||
from wrolpi.collections.reorganize import execute_batch_reorganization
|
||||
|
||||
session = request.ctx.session
|
||||
result = execute_batch_reorganization('domain', session)
|
||||
return json_response(result)
|
||||
|
||||
|
||||
@collection_bp.get('/reorganize/batch/status/<batch_job_id:str>')
|
||||
@openapi.summary('Get batch reorganization status')
|
||||
@openapi.response(HTTPStatus.OK, schema.BatchReorganizationStatusResponse,
|
||||
description="Batch reorganization status")
|
||||
async def get_batch_reorganization_status_endpoint(_: Request, batch_job_id: str):
|
||||
"""
|
||||
Get status of a batch reorganization job.
|
||||
|
||||
Returns:
|
||||
- status: 'pending' | 'running' | 'complete' | 'failed' | 'unknown'
|
||||
- total_collections: Total collections in batch
|
||||
- completed_collections: Number completed so far
|
||||
- current_collection: Info about currently processing collection
|
||||
- overall_percent: Overall completion percentage
|
||||
- completed: List of completed collection info
|
||||
- failed_collection: Info about failed collection (if any)
|
||||
- error: Error message if failed
|
||||
"""
|
||||
from wrolpi.collections.reorganize import get_batch_reorganization_status
|
||||
|
||||
status = get_batch_reorganization_status(batch_job_id)
|
||||
return json_response(status)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
from wrolpi.errors import APIError
|
||||
|
||||
__all__ = ['UnknownCollection']
|
||||
__all__ = ['UnknownCollection', 'ReorganizationConflict']
|
||||
|
||||
|
||||
class UnknownCollection(APIError):
|
||||
"""Cannot find Collection"""
|
||||
code = 'UNKNOWN_COLLECTION'
|
||||
summary = 'Cannot find Collection'
|
||||
|
||||
|
||||
class ReorganizationConflict(APIError):
|
||||
"""Reorganization would cause filename conflicts"""
|
||||
code = 'REORGANIZATION_CONFLICT'
|
||||
summary = 'Reorganization would cause filename conflicts'
|
||||
|
|
|
|||
|
|
@ -90,6 +90,10 @@ class Collection(ModelHelper, Base):
|
|||
tag_id = Column(Integer, ForeignKey('tag.id'))
|
||||
tag = relationship('Tag', primaryjoin='Collection.tag_id==Tag.id')
|
||||
|
||||
# Stores the file_name_format used when files were last organized
|
||||
# Used to detect when reorganization is needed after config changes
|
||||
file_format = Column(String, nullable=True)
|
||||
|
||||
created_date = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
|
||||
# Columns updated by triggers (similar to Channel)
|
||||
|
|
@ -119,6 +123,36 @@ class Collection(ModelHelper, Base):
|
|||
def tag_name(self) -> str | None:
|
||||
return self.tag.name if self.tag else None
|
||||
|
||||
@property
|
||||
def needs_reorganization(self) -> bool:
|
||||
"""Returns True if collection's file_format differs from current config.
|
||||
|
||||
When a user changes the file_name_format in their config, existing files
|
||||
remain in their old locations. This property detects that mismatch so the
|
||||
UI can offer to reorganize files.
|
||||
|
||||
Returns True when:
|
||||
- file_format is None (collection has never been synced to config)
|
||||
- file_format differs from current config format
|
||||
"""
|
||||
current = self._get_current_file_format()
|
||||
if not current:
|
||||
return False
|
||||
if not self.file_format:
|
||||
# Never been reorganized - needs to sync to current config
|
||||
return True
|
||||
return current != self.file_format
|
||||
|
||||
def _get_current_file_format(self) -> str | None:
|
||||
"""Get current file_name_format from config based on collection kind."""
|
||||
if self.kind == 'channel':
|
||||
from modules.videos.lib import get_videos_downloader_config
|
||||
return get_videos_downloader_config().file_name_format
|
||||
elif self.kind == 'domain':
|
||||
from modules.archive.lib import get_archive_downloader_config
|
||||
return get_archive_downloader_config().file_name_format
|
||||
return None
|
||||
|
||||
@property
|
||||
def location(self) -> str:
|
||||
"""The href of the collection in the App."""
|
||||
|
|
@ -175,6 +209,9 @@ class Collection(ModelHelper, Base):
|
|||
|
||||
config['tag_name'] = self.tag_name
|
||||
|
||||
if self.file_format:
|
||||
config['file_format'] = self.file_format
|
||||
|
||||
# Include downloads if any exist
|
||||
if self.downloads:
|
||||
config['downloads'] = [
|
||||
|
|
@ -251,6 +288,7 @@ class Collection(ModelHelper, Base):
|
|||
description = data.get('description')
|
||||
directory = data.get('directory')
|
||||
tag_name = data.get('tag_name')
|
||||
file_format = data.get('file_format')
|
||||
kind = (data.get('kind') or 'channel').strip()
|
||||
# Validate known kinds (forward-compatible: allow future values)
|
||||
if kind not in {'channel', 'domain'}:
|
||||
|
|
@ -280,6 +318,7 @@ class Collection(ModelHelper, Base):
|
|||
collection.description = description
|
||||
collection.directory = directory
|
||||
collection.kind = kind
|
||||
collection.file_format = file_format
|
||||
|
||||
if tag_name:
|
||||
tag = Tag.get_by_name(session, tag_name)
|
||||
|
|
@ -298,6 +337,7 @@ class Collection(ModelHelper, Base):
|
|||
description=description,
|
||||
directory=directory,
|
||||
kind=kind,
|
||||
file_format=file_format,
|
||||
)
|
||||
|
||||
if tag_name:
|
||||
|
|
@ -387,6 +427,7 @@ class Collection(ModelHelper, Base):
|
|||
description = data.get('description')
|
||||
directory = data.get('directory')
|
||||
tag_name = data.get('tag_name')
|
||||
file_format = data.get('file_format')
|
||||
kind = (data.get('kind') or 'channel').strip()
|
||||
|
||||
# Validate known kinds
|
||||
|
|
@ -421,6 +462,7 @@ class Collection(ModelHelper, Base):
|
|||
collection.description = description
|
||||
collection.directory = directory
|
||||
collection.kind = kind
|
||||
collection.file_format = file_format
|
||||
|
||||
if tag_name:
|
||||
tag = tags_by_name.get(tag_name)
|
||||
|
|
@ -439,6 +481,7 @@ class Collection(ModelHelper, Base):
|
|||
description=description,
|
||||
directory=directory,
|
||||
kind=kind,
|
||||
file_format=file_format,
|
||||
)
|
||||
|
||||
if tag_name:
|
||||
|
|
@ -824,6 +867,8 @@ class Collection(ModelHelper, Base):
|
|||
'item_count': self.item_count,
|
||||
'total_size': self.total_size,
|
||||
'downloads': self.downloads,
|
||||
'file_format': self.file_format,
|
||||
'needs_reorganization': self.needs_reorganization,
|
||||
}
|
||||
|
||||
def format_directory(self, tag_name: Optional[str]) -> pathlib.Path:
|
||||
|
|
@ -898,13 +943,14 @@ class Collection(ModelHelper, Base):
|
|||
Absolute path to the collection's directory
|
||||
"""
|
||||
if self.directory:
|
||||
return get_media_directory() / self.directory
|
||||
# Return absolute path (joining with media_directory handles both relative and absolute paths)
|
||||
return self.directory if self.directory.is_absolute() else get_media_directory() / self.directory
|
||||
|
||||
# First use - format and save
|
||||
tag_name = tag_name or (self.tag.name if self.tag else None)
|
||||
destination = self.format_destination(tag_name)
|
||||
|
||||
self.directory = get_relative_to_media_directory(destination)
|
||||
self.directory = destination # Store absolute path; MediaPathType handles conversion
|
||||
session.commit()
|
||||
|
||||
# Trigger config save for persistence
|
||||
|
|
|
|||
801
wrolpi/collections/reorganize.py
Normal file
801
wrolpi/collections/reorganize.py
Normal file
|
|
@ -0,0 +1,801 @@
|
|||
"""
|
||||
Collection Reorganization Module
|
||||
|
||||
Provides functionality to reorganize files within Collections (Domains/Channels)
|
||||
when the file_name_format config changes.
|
||||
|
||||
Example: User changes archive format from '%(title)s.%(ext)s' to '%(download_year)s/%(title)s.%(ext)s'
|
||||
- Before: 'My Article.html'
|
||||
- After: '2025/My Article.html'
|
||||
"""
|
||||
import pathlib
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
|
||||
from wrolpi.common import logger, get_media_directory
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
from .errors import UnknownCollection
|
||||
from .models import Collection
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = [
|
||||
'ReorganizationPreview',
|
||||
'get_reorganization_preview',
|
||||
'execute_reorganization',
|
||||
'get_collections_needing_reorganization',
|
||||
'execute_batch_reorganization',
|
||||
'get_batch_reorganization_status',
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReorganizationPreview:
|
||||
"""Preview of a collection reorganization operation.
|
||||
|
||||
Note: For performance reasons, files_needing_move equals total_files.
|
||||
We assume all files need moving since computing actual moves for every
|
||||
file is expensive and would defeat the purpose of this preview.
|
||||
"""
|
||||
collection_id: int
|
||||
collection_name: str
|
||||
total_files: int # Total FileGroups in collection
|
||||
files_needing_move: int # Equals total_files (assumed all need moving)
|
||||
sample_moves: List[dict] # Sample of moves: [{old_path, new_path}, ...]
|
||||
new_file_format: str # The format that will be applied
|
||||
current_file_format: Optional[str] # The format currently stored on collection
|
||||
|
||||
|
||||
def _get_format_required_variables(file_format: str) -> set:
|
||||
"""Parse a format string and return the set of variable names it uses.
|
||||
|
||||
Example: '%(upload_year)s/%(title)s.%(ext)s' -> {'upload_year', 'title', 'ext'}
|
||||
"""
|
||||
import re
|
||||
# Match %(variable_name)s patterns
|
||||
return set(re.findall(r'%\(([^)]+)\)s', file_format))
|
||||
|
||||
|
||||
def _video_has_required_metadata(video, file_format: str) -> bool:
|
||||
"""Check if video has the metadata required by the format string.
|
||||
|
||||
Returns False only if the format requires a variable that the video cannot provide.
|
||||
"""
|
||||
required = _get_format_required_variables(file_format)
|
||||
|
||||
file_group = video.file_group
|
||||
info_json = video.get_info_json() or {}
|
||||
|
||||
# Check each potentially-missing variable
|
||||
# title and ext are always available (have fallbacks)
|
||||
has_upload_date = bool(
|
||||
file_group.published_datetime or info_json.get('upload_date')
|
||||
)
|
||||
has_source_id = bool(video.source_id or info_json.get('id'))
|
||||
has_uploader = bool(info_json.get('uploader') or info_json.get('channel'))
|
||||
|
||||
# Map format variables to their metadata availability
|
||||
# Variables that depend on upload_date
|
||||
date_vars = {'upload_year', 'upload_month', 'upload_date', 'upload_date>%Y', 'upload_date>%m'}
|
||||
# Variables that depend on source_id
|
||||
id_vars = {'id'}
|
||||
# Variables that depend on uploader
|
||||
uploader_vars = {'uploader', 'channel'}
|
||||
|
||||
for var in required:
|
||||
if var in date_vars and not has_upload_date:
|
||||
return False
|
||||
if var in id_vars and not has_source_id:
|
||||
return False
|
||||
if var in uploader_vars and not has_uploader:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _compute_new_path_for_video(
|
||||
video,
|
||||
collection_directory: pathlib.Path,
|
||||
file_format: str = None,
|
||||
) -> Optional[pathlib.Path]:
|
||||
"""Compute the new path for a video based on current config.
|
||||
|
||||
Args:
|
||||
video: Video model instance
|
||||
collection_directory: Base directory for the collection
|
||||
file_format: Optional format string override, defaults to config value
|
||||
|
||||
Returns None if the video lacks required metadata for the configured format.
|
||||
"""
|
||||
from modules.videos.lib import format_video_filename, get_videos_downloader_config
|
||||
|
||||
if file_format is None:
|
||||
config = get_videos_downloader_config()
|
||||
file_format = config.file_name_format
|
||||
|
||||
if not _video_has_required_metadata(video, file_format):
|
||||
logger.info(f'Skipping video {video.id}: missing metadata required by format "{file_format}"')
|
||||
return None
|
||||
try:
|
||||
new_filename = format_video_filename(video, file_format)
|
||||
return collection_directory / new_filename
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to compute new path for video {video.id}: {e}')
|
||||
return None
|
||||
|
||||
|
||||
def _archive_has_required_metadata(archive, file_format: str) -> bool:
|
||||
"""Check if archive has the metadata required by the format string.
|
||||
|
||||
Returns False only if the format requires a variable that the archive cannot provide.
|
||||
Archives have robust fallbacks for most variables (dates fallback to now(),
|
||||
title to 'untitled'). Only domain can be empty when URL is missing.
|
||||
"""
|
||||
required = _get_format_required_variables(file_format)
|
||||
|
||||
file_group = archive.file_group
|
||||
|
||||
# Domain requires a URL
|
||||
has_domain = bool(file_group.url)
|
||||
|
||||
# Domain-related variables
|
||||
domain_vars = {'domain'}
|
||||
|
||||
for var in required:
|
||||
if var in domain_vars and not has_domain:
|
||||
return False
|
||||
|
||||
# All other variables (title, download_*, ext) always have fallbacks
|
||||
return True
|
||||
|
||||
|
||||
def _compute_new_path_for_archive(
|
||||
archive,
|
||||
collection_directory: pathlib.Path,
|
||||
file_format: str = None,
|
||||
) -> Optional[pathlib.Path]:
|
||||
"""Compute the new path for an archive based on current config.
|
||||
|
||||
Args:
|
||||
archive: Archive model instance
|
||||
collection_directory: Base directory for the collection
|
||||
file_format: Optional format string override, defaults to config value
|
||||
|
||||
Returns None if the archive lacks required metadata for the configured format.
|
||||
"""
|
||||
from modules.archive.lib import format_archive_filename_from_archive, get_archive_downloader_config
|
||||
|
||||
if file_format is None:
|
||||
config = get_archive_downloader_config()
|
||||
file_format = config.file_name_format
|
||||
|
||||
if not _archive_has_required_metadata(archive, file_format):
|
||||
logger.info(f'Skipping archive {archive.id}: missing metadata required by format "{file_format}"')
|
||||
return None
|
||||
|
||||
try:
|
||||
new_filename = format_archive_filename_from_archive(archive, file_format)
|
||||
return collection_directory / new_filename
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to compute new path for archive {archive.id}: {e}')
|
||||
return None
|
||||
|
||||
|
||||
def get_reorganization_preview(
|
||||
collection_id: int,
|
||||
session: Session = None,
|
||||
sample_size: int = 10,
|
||||
exact_count: bool = False,
|
||||
) -> ReorganizationPreview:
|
||||
"""Get preview with sample of files to be renamed.
|
||||
|
||||
Does NOT compute all moves by default - only samples for preview display.
|
||||
|
||||
Args:
|
||||
collection_id: The collection to preview reorganization for
|
||||
session: Database session (optional, will create one if not provided)
|
||||
sample_size: Number of sample moves to include in preview
|
||||
exact_count: If True, compute the actual count of files needing moves
|
||||
(more expensive but accurate for individual collection preview)
|
||||
|
||||
Returns:
|
||||
ReorganizationPreview with counts and sample moves
|
||||
|
||||
Raises:
|
||||
UnknownCollection: If collection not found
|
||||
"""
|
||||
if session is None:
|
||||
with get_db_session() as session:
|
||||
return get_reorganization_preview(collection_id, session, sample_size, exact_count)
|
||||
|
||||
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
|
||||
if not collection:
|
||||
raise UnknownCollection(f"Collection with ID {collection_id} not found")
|
||||
|
||||
if not collection.directory:
|
||||
raise ValueError(f"Collection '{collection.name}' has no directory. Cannot reorganize.")
|
||||
|
||||
collection_directory = pathlib.Path(collection.directory)
|
||||
if not collection_directory.is_absolute():
|
||||
collection_directory = get_media_directory() / collection_directory
|
||||
|
||||
# Get the current file format from config
|
||||
current_config_format = collection._get_current_file_format()
|
||||
|
||||
sample_moves = []
|
||||
files_needing_move = 0
|
||||
total_files = 0
|
||||
|
||||
if collection.kind == 'channel':
|
||||
from modules.videos.models import Video, Channel
|
||||
|
||||
# Get the channel for this collection
|
||||
channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
|
||||
if not channel:
|
||||
return ReorganizationPreview(
|
||||
collection_id=collection_id,
|
||||
collection_name=collection.name,
|
||||
total_files=0,
|
||||
files_needing_move=0,
|
||||
sample_moves=[],
|
||||
new_file_format=current_config_format or '',
|
||||
current_file_format=collection.file_format,
|
||||
)
|
||||
|
||||
# Use COUNT for total_files (fast)
|
||||
total_files = session.query(func.count(Video.id)).filter_by(channel_id=channel.id).scalar()
|
||||
|
||||
if exact_count:
|
||||
# Compute actual count by building full move mappings
|
||||
move_mappings = _build_move_mappings_for_channel(collection, session)
|
||||
files_needing_move = len(move_mappings)
|
||||
else:
|
||||
# Assume all files need moving (computing actual count is expensive)
|
||||
files_needing_move = total_files
|
||||
|
||||
# Only load limited videos for sample search
|
||||
videos = session.query(Video).options(
|
||||
joinedload(Video.file_group)
|
||||
).filter_by(channel_id=channel.id).limit(10).all()
|
||||
|
||||
for video in videos:
|
||||
if not video.file_group or not video.file_group.primary_path:
|
||||
continue
|
||||
|
||||
current_path = pathlib.Path(video.file_group.primary_path)
|
||||
new_path = _compute_new_path_for_video(video, collection_directory)
|
||||
|
||||
if new_path and current_path != new_path:
|
||||
# Get relative paths for display
|
||||
media_dir = get_media_directory()
|
||||
try:
|
||||
old_relative = str(current_path.relative_to(media_dir))
|
||||
except ValueError:
|
||||
old_relative = str(current_path)
|
||||
try:
|
||||
new_relative = str(new_path.relative_to(media_dir))
|
||||
except ValueError:
|
||||
new_relative = str(new_path)
|
||||
|
||||
sample_moves.append({
|
||||
'old_path': old_relative,
|
||||
'new_path': new_relative,
|
||||
})
|
||||
if len(sample_moves) >= sample_size:
|
||||
break
|
||||
|
||||
elif collection.kind == 'domain':
|
||||
from modules.archive.models import Archive
|
||||
|
||||
# Use COUNT for total_files (fast)
|
||||
total_files = session.query(func.count(Archive.id)).filter_by(collection_id=collection_id).scalar()
|
||||
|
||||
if exact_count:
|
||||
# Compute actual count by building full move mappings
|
||||
move_mappings = _build_move_mappings_for_domain(collection, session)
|
||||
files_needing_move = len(move_mappings)
|
||||
else:
|
||||
# Assume all files need moving (computing actual count is expensive)
|
||||
files_needing_move = total_files
|
||||
|
||||
# Only load limited archives for sample search
|
||||
archives = session.query(Archive).options(
|
||||
joinedload(Archive.file_group)
|
||||
).filter_by(collection_id=collection_id).limit(10).all()
|
||||
|
||||
for archive in archives:
|
||||
if not archive.file_group or not archive.file_group.primary_path:
|
||||
continue
|
||||
|
||||
current_path = pathlib.Path(archive.file_group.primary_path)
|
||||
new_path = _compute_new_path_for_archive(archive, collection_directory)
|
||||
|
||||
if new_path and current_path != new_path:
|
||||
# Get relative paths for display
|
||||
media_dir = get_media_directory()
|
||||
try:
|
||||
old_relative = str(current_path.relative_to(media_dir))
|
||||
except ValueError:
|
||||
old_relative = str(current_path)
|
||||
try:
|
||||
new_relative = str(new_path.relative_to(media_dir))
|
||||
except ValueError:
|
||||
new_relative = str(new_path)
|
||||
|
||||
sample_moves.append({
|
||||
'old_path': old_relative,
|
||||
'new_path': new_relative,
|
||||
})
|
||||
if len(sample_moves) >= sample_size:
|
||||
break
|
||||
|
||||
return ReorganizationPreview(
|
||||
collection_id=collection_id,
|
||||
collection_name=collection.name,
|
||||
total_files=total_files,
|
||||
files_needing_move=files_needing_move,
|
||||
sample_moves=sample_moves,
|
||||
new_file_format=current_config_format or '',
|
||||
current_file_format=collection.file_format,
|
||||
)
|
||||
|
||||
|
||||
def _build_move_mappings_for_channel(
|
||||
collection: Collection,
|
||||
session: Session,
|
||||
) -> List[Tuple[pathlib.Path, pathlib.Path]]:
|
||||
"""Build move mappings for a channel collection."""
|
||||
from modules.videos.models import Video, Channel
|
||||
|
||||
collection_directory = pathlib.Path(collection.directory)
|
||||
if not collection_directory.is_absolute():
|
||||
collection_directory = get_media_directory() / collection_directory
|
||||
|
||||
channel = session.query(Channel).filter_by(collection_id=collection.id).one_or_none()
|
||||
if not channel:
|
||||
return []
|
||||
|
||||
move_mappings = []
|
||||
videos = session.query(Video).options(
|
||||
joinedload(Video.file_group)
|
||||
).filter_by(channel_id=channel.id).all()
|
||||
|
||||
for video in videos:
|
||||
if not video.file_group or not video.file_group.primary_path:
|
||||
continue
|
||||
|
||||
current_path = pathlib.Path(video.file_group.primary_path)
|
||||
new_path = _compute_new_path_for_video(video, collection_directory)
|
||||
|
||||
if new_path and current_path != new_path:
|
||||
move_mappings.append((current_path, new_path))
|
||||
|
||||
return move_mappings
|
||||
|
||||
|
||||
def _build_move_mappings_for_domain(
|
||||
collection: Collection,
|
||||
session: Session,
|
||||
) -> List[Tuple[pathlib.Path, pathlib.Path]]:
|
||||
"""Build move mappings for a domain collection."""
|
||||
from modules.archive.models import Archive
|
||||
|
||||
collection_directory = pathlib.Path(collection.directory)
|
||||
if not collection_directory.is_absolute():
|
||||
collection_directory = get_media_directory() / collection_directory
|
||||
|
||||
move_mappings = []
|
||||
archives = session.query(Archive).options(
|
||||
joinedload(Archive.file_group)
|
||||
).filter_by(collection_id=collection.id).all()
|
||||
|
||||
for archive in archives:
|
||||
if not archive.file_group or not archive.file_group.primary_path:
|
||||
continue
|
||||
|
||||
current_path = pathlib.Path(archive.file_group.primary_path)
|
||||
new_path = _compute_new_path_for_archive(archive, collection_directory)
|
||||
|
||||
if new_path and current_path != new_path:
|
||||
move_mappings.append((current_path, new_path))
|
||||
|
||||
return move_mappings
|
||||
|
||||
|
||||
def _check_for_destination_conflicts(
|
||||
move_mappings: List[Tuple[pathlib.Path, pathlib.Path]],
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Check for duplicate destination paths in move mappings.
|
||||
|
||||
Raises:
|
||||
ReorganizationConflict: If two or more files would move to the same destination
|
||||
"""
|
||||
from wrolpi.collections.errors import ReorganizationConflict
|
||||
|
||||
dest_to_sources = {}
|
||||
for source, dest in move_mappings:
|
||||
if dest in dest_to_sources:
|
||||
dest_to_sources[dest].append(source)
|
||||
else:
|
||||
dest_to_sources[dest] = [source]
|
||||
|
||||
conflicts = {dest: sources for dest, sources in dest_to_sources.items() if len(sources) > 1}
|
||||
|
||||
if conflicts:
|
||||
conflict_details = []
|
||||
for dest, sources in conflicts.items():
|
||||
source_names = [str(s.name) for s in sources]
|
||||
conflict_details.append(f" {dest.name} <- {', '.join(source_names)}")
|
||||
|
||||
raise ReorganizationConflict(
|
||||
f"Reorganization of '{collection_name}' would cause {len(conflicts)} filename conflict(s):\n" +
|
||||
"\n".join(conflict_details)
|
||||
)
|
||||
|
||||
|
||||
def execute_reorganization(
|
||||
collection_id: int,
|
||||
session: Session = None,
|
||||
) -> str:
|
||||
"""Execute reorganization, returns job_id for tracking.
|
||||
|
||||
Iterates ALL FileGroups and queues moves to FileWorker.
|
||||
|
||||
Args:
|
||||
collection_id: The collection to reorganize
|
||||
session: Database session (optional)
|
||||
|
||||
Returns:
|
||||
job_id for tracking progress via FileWorker status
|
||||
|
||||
Raises:
|
||||
UnknownCollection: If collection not found
|
||||
"""
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
if session is None:
|
||||
with get_db_session(commit=True) as session:
|
||||
return execute_reorganization(collection_id, session)
|
||||
|
||||
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
|
||||
if not collection:
|
||||
raise UnknownCollection(f"Collection with ID {collection_id} not found")
|
||||
|
||||
if not collection.directory:
|
||||
raise ValueError(f"Collection '{collection.name}' has no directory. Cannot reorganize.")
|
||||
|
||||
# Build move mappings based on collection kind
|
||||
if collection.kind == 'channel':
|
||||
move_mappings = _build_move_mappings_for_channel(collection, session)
|
||||
elif collection.kind == 'domain':
|
||||
move_mappings = _build_move_mappings_for_domain(collection, session)
|
||||
else:
|
||||
raise ValueError(f"Unsupported collection kind: {collection.kind}")
|
||||
|
||||
# Check for filename conflicts before proceeding
|
||||
_check_for_destination_conflicts(move_mappings, collection.name)
|
||||
|
||||
if not move_mappings:
|
||||
logger.info(f'No files need reorganization for collection {collection.name}')
|
||||
# Still update the file_format to current config
|
||||
current_format = collection._get_current_file_format()
|
||||
if current_format:
|
||||
collection.file_format = current_format
|
||||
session.commit()
|
||||
|
||||
# Trigger config save
|
||||
if collection.kind == 'domain':
|
||||
from modules.archive.lib import save_domains_config
|
||||
save_domains_config.activate_switch()
|
||||
elif collection.kind == 'channel':
|
||||
from modules.videos.lib import save_channels_config
|
||||
save_channels_config.activate_switch()
|
||||
|
||||
return ''
|
||||
|
||||
logger.info(f'Queueing reorganization for collection {collection.name}: {len(move_mappings)} files')
|
||||
|
||||
# Get the current file format to update AFTER successful completion
|
||||
# (deferred update enables retry on failure - file_format only updates on success)
|
||||
pending_file_format = collection._get_current_file_format()
|
||||
|
||||
# Queue the reorganization with FileWorker
|
||||
# The pending_file_format will be set by handle_reorganize after successful completion
|
||||
job_id = file_worker.queue_reorganize(
|
||||
move_mappings,
|
||||
collection_id=collection_id,
|
||||
collection_kind=collection.kind,
|
||||
pending_file_format=pending_file_format,
|
||||
)
|
||||
|
||||
return job_id
|
||||
|
||||
|
||||
def get_reorganization_status(job_id: str) -> dict:
|
||||
"""Get status of an in-flight reorganization.
|
||||
|
||||
Args:
|
||||
job_id: The job ID returned by execute_reorganization
|
||||
|
||||
Returns:
|
||||
Dict with status information:
|
||||
- status: 'pending' | 'running' | 'complete' | 'failed'
|
||||
- total: Total files to move
|
||||
- completed: Files moved so far
|
||||
- percent: Completion percentage
|
||||
- error: Error message if failed
|
||||
"""
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
try:
|
||||
job_status = file_worker.get_job_status(job_id)
|
||||
except RuntimeError:
|
||||
return {'status': 'unknown', 'error': f'Job {job_id} not found'}
|
||||
|
||||
# Get current worker status if this job is running
|
||||
worker_status = file_worker.status
|
||||
|
||||
if job_status == 'complete':
|
||||
return {
|
||||
'status': 'complete',
|
||||
'total': worker_status.get('operation_total', 0),
|
||||
'completed': worker_status.get('operation_total', 0),
|
||||
'percent': 100,
|
||||
'error': None,
|
||||
}
|
||||
elif job_status == 'pending':
|
||||
return {
|
||||
'status': 'pending',
|
||||
'total': 0,
|
||||
'completed': 0,
|
||||
'percent': 0,
|
||||
'error': None,
|
||||
}
|
||||
else:
|
||||
# Job is running
|
||||
return {
|
||||
'status': worker_status.get('status', 'running'),
|
||||
'total': worker_status.get('operation_total', 0),
|
||||
'completed': worker_status.get('operation_processed', 0),
|
||||
'percent': worker_status.get('operation_percent', 0),
|
||||
'error': worker_status.get('error'),
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Batch Reorganization Functions
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def get_collections_needing_reorganization(
|
||||
kind: str,
|
||||
session: Session = None,
|
||||
sample_size: int = 1,
|
||||
) -> dict:
|
||||
"""Get all collections of a kind that need reorganization with sample previews.
|
||||
|
||||
Args:
|
||||
kind: 'channel' or 'domain'
|
||||
session: Database session (optional)
|
||||
sample_size: Number of sample moves per collection (default 1)
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- collections: List of collection info dicts
|
||||
- total_collections: Total count needing reorganization
|
||||
- total_files_needing_move: Sum of all files needing reorganization
|
||||
- new_file_format: The config format that will be applied
|
||||
"""
|
||||
if session is None:
|
||||
with get_db_session() as session:
|
||||
return get_collections_needing_reorganization(kind, session, sample_size)
|
||||
|
||||
# Query all collections of this kind with directories
|
||||
collections = session.query(Collection).filter(
|
||||
Collection.kind == kind,
|
||||
Collection.directory.isnot(None),
|
||||
).all()
|
||||
|
||||
# Filter to only those needing reorganization
|
||||
collections_needing_reorg = [c for c in collections if c.needs_reorganization]
|
||||
|
||||
# Get current config format
|
||||
current_format = ''
|
||||
if kind == 'channel':
|
||||
from modules.videos.lib import get_videos_downloader_config
|
||||
current_format = get_videos_downloader_config().file_name_format
|
||||
elif kind == 'domain':
|
||||
from modules.archive.lib import get_archive_downloader_config
|
||||
current_format = get_archive_downloader_config().file_name_format
|
||||
|
||||
result_collections = []
|
||||
total_files_needing_move = 0
|
||||
|
||||
for collection in collections_needing_reorg:
|
||||
# Get preview for each collection
|
||||
try:
|
||||
preview = get_reorganization_preview(collection.id, session, sample_size=sample_size)
|
||||
sample_move = preview.sample_moves[0] if preview.sample_moves else None
|
||||
result_collections.append({
|
||||
'collection_id': collection.id,
|
||||
'collection_name': collection.name,
|
||||
'total_files': preview.total_files,
|
||||
'files_needing_move': preview.files_needing_move,
|
||||
'sample_move': sample_move,
|
||||
})
|
||||
total_files_needing_move += preview.files_needing_move
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to get preview for collection {collection.id}: {e}')
|
||||
result_collections.append({
|
||||
'collection_id': collection.id,
|
||||
'collection_name': collection.name,
|
||||
'total_files': 0,
|
||||
'files_needing_move': 0,
|
||||
'sample_move': None,
|
||||
})
|
||||
|
||||
return {
|
||||
'collections': result_collections,
|
||||
'total_collections': len(result_collections),
|
||||
'total_files_needing_move': total_files_needing_move,
|
||||
'new_file_format': current_format,
|
||||
}
|
||||
|
||||
|
||||
def execute_batch_reorganization(kind: str, session: Session = None) -> dict:
|
||||
"""Execute batch reorganization for all collections of a kind that need it.
|
||||
|
||||
This queues a batch reorganization job that processes collections sequentially.
|
||||
Each collection is reorganized one at a time to avoid system overload.
|
||||
If any collection fails, the batch stops and reports which collection failed.
|
||||
|
||||
Args:
|
||||
kind: 'channel' or 'domain'
|
||||
session: Database session (optional)
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- batch_job_id: Job ID for tracking progress
|
||||
- message: Status message
|
||||
- collection_count: Number of collections to process
|
||||
"""
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
if session is None:
|
||||
with get_db_session() as session:
|
||||
return execute_batch_reorganization(kind, session)
|
||||
|
||||
# Get collections needing reorganization
|
||||
reorg_info = get_collections_needing_reorganization(kind, session, sample_size=0)
|
||||
collections_to_process = reorg_info['collections']
|
||||
|
||||
if not collections_to_process:
|
||||
logger.info(f'No {kind} collections need reorganization')
|
||||
return {
|
||||
'batch_job_id': '',
|
||||
'message': f'No {kind} collections need reorganization',
|
||||
'collection_count': 0,
|
||||
}
|
||||
|
||||
# Extract collection IDs for the batch job
|
||||
collection_ids = [c['collection_id'] for c in collections_to_process]
|
||||
|
||||
logger.info(f'Queueing batch reorganization for {len(collection_ids)} {kind} collections')
|
||||
|
||||
# Queue the batch reorganization
|
||||
batch_job_id = file_worker.queue_batch_reorganize(collection_ids, kind)
|
||||
|
||||
return {
|
||||
'batch_job_id': batch_job_id,
|
||||
'message': 'Batch reorganization started',
|
||||
'collection_count': len(collection_ids),
|
||||
}
|
||||
|
||||
|
||||
def get_batch_reorganization_status(batch_job_id: str) -> dict:
|
||||
"""Get status of a batch reorganization job.
|
||||
|
||||
Args:
|
||||
batch_job_id: The batch job ID returned by execute_batch_reorganization
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- status: 'pending' | 'running' | 'complete' | 'failed' | 'unknown'
|
||||
- total_collections: Total collections in batch
|
||||
- completed_collections: Number completed so far
|
||||
- current_collection: Info about currently processing collection (or None)
|
||||
- overall_percent: Overall completion percentage
|
||||
- completed: List of completed collection info
|
||||
- failed_collection: Info about failed collection (or None)
|
||||
- error: Error message if failed
|
||||
"""
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
try:
|
||||
job_status = file_worker.get_job_status(batch_job_id)
|
||||
except RuntimeError:
|
||||
return {
|
||||
'status': 'unknown',
|
||||
'error': f'Batch job {batch_job_id} not found',
|
||||
'total_collections': 0,
|
||||
'completed_collections': 0,
|
||||
'current_collection': None,
|
||||
'overall_percent': 0,
|
||||
'completed': [],
|
||||
'failed_collection': None,
|
||||
}
|
||||
|
||||
# Get batch-specific status from worker
|
||||
worker_status = file_worker.status
|
||||
batch_status = worker_status.get('batch_status', {})
|
||||
|
||||
if job_status == 'complete':
|
||||
completed_list = batch_status.get('completed', [])
|
||||
return {
|
||||
'status': 'complete',
|
||||
'total_collections': batch_status.get('total_collections', 0),
|
||||
'completed_collections': len(completed_list),
|
||||
'current_collection': None,
|
||||
'overall_percent': 100,
|
||||
'completed': completed_list,
|
||||
'failed_collection': None,
|
||||
'error': None,
|
||||
}
|
||||
elif job_status == 'pending':
|
||||
return {
|
||||
'status': 'pending',
|
||||
'total_collections': batch_status.get('total_collections', 0),
|
||||
'completed_collections': 0,
|
||||
'current_collection': None,
|
||||
'overall_percent': 0,
|
||||
'completed': [],
|
||||
'failed_collection': None,
|
||||
'error': None,
|
||||
}
|
||||
elif job_status == 'failed':
|
||||
completed_list = batch_status.get('completed', [])
|
||||
return {
|
||||
'status': 'failed',
|
||||
'total_collections': batch_status.get('total_collections', 0),
|
||||
'completed_collections': len(completed_list),
|
||||
'current_collection': None,
|
||||
'overall_percent': batch_status.get('overall_percent', 0),
|
||||
'completed': completed_list,
|
||||
'failed_collection': batch_status.get('failed_collection'),
|
||||
'error': batch_status.get('error'),
|
||||
}
|
||||
else:
|
||||
# Job is running
|
||||
completed_list = batch_status.get('completed', [])
|
||||
current = batch_status.get('current_collection')
|
||||
total = batch_status.get('total_collections', 1)
|
||||
|
||||
# Get fresh file-level progress from worker status (not from batch_status which may be stale)
|
||||
current_percent = worker_status.get('operation_percent', 0)
|
||||
|
||||
# Update current collection dict with fresh values if it exists
|
||||
if current:
|
||||
current = dict(current) # Copy to avoid mutating the original
|
||||
current['percent'] = current_percent
|
||||
current['completed'] = worker_status.get('operation_processed', 0)
|
||||
current['total'] = worker_status.get('operation_total', 0)
|
||||
|
||||
# Calculate overall percent: (completed + current_progress) / total
|
||||
completed_count = len(completed_list)
|
||||
overall_percent = int(((completed_count + current_percent / 100) / total) * 100) if total > 0 else 0
|
||||
|
||||
return {
|
||||
'status': 'running',
|
||||
'total_collections': total,
|
||||
'completed_collections': completed_count,
|
||||
'current_collection': current,
|
||||
'overall_percent': overall_percent,
|
||||
'completed': completed_list,
|
||||
'failed_collection': None,
|
||||
'error': None,
|
||||
}
|
||||
|
|
@ -52,3 +52,88 @@ class CollectionTagInfoResponse:
|
|||
suggested_directory: str
|
||||
conflict: bool
|
||||
conflict_message: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReorganizationPreviewResponse:
|
||||
"""Response for reorganization preview."""
|
||||
collection_id: int
|
||||
collection_name: str
|
||||
total_files: int
|
||||
files_needing_move: int
|
||||
sample_moves: List[dict]
|
||||
new_file_format: str
|
||||
current_file_format: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReorganizationExecuteResponse:
|
||||
"""Response for reorganization execution."""
|
||||
job_id: str
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReorganizationStatusResponse:
|
||||
"""Response for reorganization status."""
|
||||
status: str
|
||||
total: int
|
||||
completed: int
|
||||
percent: int
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Batch Reorganization Schemas
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchCollectionInfo:
|
||||
"""Info about a single collection in batch reorganization."""
|
||||
collection_id: int
|
||||
collection_name: str
|
||||
total_files: int
|
||||
files_needing_move: int
|
||||
sample_move: Optional[dict] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchReorganizationListResponse:
|
||||
"""Response for listing collections needing batch reorganization."""
|
||||
collections: List[dict]
|
||||
total_collections: int
|
||||
total_files_needing_move: int
|
||||
new_file_format: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchReorganizationExecuteResponse:
|
||||
"""Response for executing batch reorganization."""
|
||||
batch_job_id: str
|
||||
message: str
|
||||
collection_count: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchCollectionProgress:
|
||||
"""Progress info for a single collection in batch reorganization."""
|
||||
id: int
|
||||
name: str
|
||||
status: str
|
||||
total: int = 0
|
||||
completed: int = 0
|
||||
percent: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchReorganizationStatusResponse:
|
||||
"""Response for batch reorganization status."""
|
||||
status: str
|
||||
total_collections: int
|
||||
completed_collections: int
|
||||
current_collection: Optional[dict]
|
||||
overall_percent: int
|
||||
completed: List[dict]
|
||||
failed_collection: Optional[dict]
|
||||
error: Optional[str] = None
|
||||
|
|
|
|||
1978
wrolpi/collections/test/test_reorganize.py
Normal file
1978
wrolpi/collections/test/test_reorganize.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1404,27 +1404,45 @@ def escape_file_name(name: str) -> str:
|
|||
return name.strip()
|
||||
|
||||
|
||||
# Maximum length is probably 255, but we need more length for large suffixes like `.readability.json`, and temporary
|
||||
# downloading suffixes from yt-dlp.
|
||||
MAXIMUM_FILE_LENGTH = 140
|
||||
# Maximum bytes for a filename. Linux ext4/xfs limit is 255 bytes.
|
||||
# We use 200 to leave room for FileGroup suffix variations (e.g., .readability.html is 17 bytes vs .html at 5 bytes).
|
||||
MAXIMUM_FILE_BYTES = 200
|
||||
|
||||
|
||||
def trim_file_name(path: str | pathlib.Path) -> str | pathlib.Path:
|
||||
"""Shorten the file name only if it is longer than the file system supports. Trim from the end of the name until
|
||||
the name is short enough (preserving any suffix)."""
|
||||
the name is short enough (preserving any suffix).
|
||||
|
||||
Uses BYTE length (not character length) because UTF-8 multi-byte characters (e.g., Devanagari, Chinese)
|
||||
can exceed the 255 byte filesystem limit even with fewer than 140 characters.
|
||||
"""
|
||||
name_type = pathlib.Path if isinstance(path, pathlib.Path) else str
|
||||
parent = path.parent if isinstance(path, pathlib.Path) else '/'.join(i for i in path.split('/')[:-1])
|
||||
path = path.name if isinstance(path, pathlib.Path) else path
|
||||
filename = path.name if isinstance(path, pathlib.Path) else path.split('/')[-1] if '/' in path else path
|
||||
|
||||
if len(path) < MAXIMUM_FILE_LENGTH:
|
||||
return name_type(path)
|
||||
filename_bytes = len(filename.encode('utf-8'))
|
||||
if filename_bytes <= MAXIMUM_FILE_BYTES:
|
||||
# No trimming needed, but preserve parent directory if present
|
||||
if parent:
|
||||
if name_type == pathlib.Path:
|
||||
return parent / filename
|
||||
return f'{parent}/{filename}'
|
||||
return name_type(filename)
|
||||
|
||||
# Don't trim the filename to exactly 256 characters.
|
||||
# Trim based on byte length, not character length.
|
||||
# This is because a FileGroup will have varying filename lengths.
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
stem, suffix = split_path_stem_and_suffix(path)
|
||||
excess = MAXIMUM_FILE_LENGTH - len(suffix)
|
||||
new_name = stem[:excess].strip() + suffix
|
||||
stem, suffix = split_path_stem_and_suffix(filename)
|
||||
suffix_bytes = len(suffix.encode('utf-8'))
|
||||
target_stem_bytes = MAXIMUM_FILE_BYTES - suffix_bytes
|
||||
|
||||
# Trim characters from end of stem until byte length fits
|
||||
trimmed_stem = stem
|
||||
while len(trimmed_stem.encode('utf-8')) > target_stem_bytes and trimmed_stem:
|
||||
trimmed_stem = trimmed_stem[:-1]
|
||||
trimmed_stem = trimmed_stem.strip()
|
||||
|
||||
new_name = trimmed_stem + suffix
|
||||
if parent:
|
||||
if name_type == pathlib.Path:
|
||||
return parent / new_name
|
||||
|
|
@ -1573,7 +1591,22 @@ def register_modeler(modeler: callable):
|
|||
return modeler
|
||||
|
||||
|
||||
async def apply_modelers():
|
||||
async def apply_modelers(progress_callback: callable = None):
|
||||
"""Run all registered modelers.
|
||||
|
||||
Args:
|
||||
progress_callback: Optional callback(processed, total) called after each modeler completes.
|
||||
"""
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
# Get initial count of unindexed files for progress tracking
|
||||
initial_unindexed = 0
|
||||
if progress_callback:
|
||||
with get_db_session() as session:
|
||||
initial_unindexed = session.query(FileGroup).filter(FileGroup.indexed != True).count()
|
||||
progress_callback(0, initial_unindexed)
|
||||
|
||||
for modeler in modelers:
|
||||
logger_.info(f'Applying modeler {modeler.__name__}')
|
||||
try:
|
||||
|
|
@ -1582,6 +1615,14 @@ async def apply_modelers():
|
|||
logger_.error(f'Modeler {modeler.__name__} raised error', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
# Update progress after each modeler
|
||||
if progress_callback and initial_unindexed > 0:
|
||||
with get_db_session() as session:
|
||||
remaining = session.query(FileGroup).filter(FileGroup.indexed != True).count()
|
||||
processed = initial_unindexed - remaining
|
||||
progress_callback(processed, initial_unindexed)
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import subprocess
|
|||
import urllib.parse
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Union, Dict, Generator, Iterable, Set
|
||||
from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set
|
||||
|
||||
import cachetools.func
|
||||
import psycopg2
|
||||
|
|
@ -724,11 +724,23 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
|
|||
curs.execute(stmt)
|
||||
|
||||
|
||||
async def apply_indexers():
|
||||
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function."""
|
||||
async def apply_indexers(progress_callback: Callable[[int, int], None] = None):
|
||||
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function.
|
||||
|
||||
Args:
|
||||
progress_callback: Optional callback(processed, total) called after each batch.
|
||||
"""
|
||||
from wrolpi.files.models import FileGroup
|
||||
refresh_logger.info('Applying indexers')
|
||||
|
||||
# Get initial count for progress tracking
|
||||
total_to_index = 0
|
||||
total_indexed = 0
|
||||
if progress_callback:
|
||||
with get_db_session() as session:
|
||||
total_to_index = session.query(FileGroup).filter(FileGroup.indexed != True).count()
|
||||
progress_callback(0, total_to_index)
|
||||
|
||||
while True:
|
||||
# Continually query for Files that have not been indexed.
|
||||
with get_db_session(commit=False) as session:
|
||||
|
|
@ -809,6 +821,11 @@ async def apply_indexers():
|
|||
if file_group:
|
||||
refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}')
|
||||
|
||||
# Update progress
|
||||
total_indexed += processed
|
||||
if progress_callback and total_to_index > 0:
|
||||
progress_callback(total_indexed, total_to_index)
|
||||
|
||||
if processed < 20:
|
||||
# Processed less than the limit, don't do the next query.
|
||||
break
|
||||
|
|
@ -1365,6 +1382,51 @@ def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path,
|
|||
curs.execute(sql)
|
||||
|
||||
|
||||
def _json_serial(obj):
|
||||
"""JSON serializer for objects not serializable by default json code."""
|
||||
import datetime
|
||||
if isinstance(obj, (datetime.datetime, datetime.date)):
|
||||
return obj.isoformat()
|
||||
raise TypeError(f"Type {type(obj)} not serializable")
|
||||
|
||||
|
||||
def _bulk_update_file_groups_reorganize(updates: List[dict]):
|
||||
"""Batch UPDATE FileGroups for reorganization using raw SQL.
|
||||
|
||||
Unlike _bulk_update_file_groups_db, this also updates the files and data JSON fields
|
||||
since reorganization changes filenames, not just directories.
|
||||
|
||||
Args:
|
||||
updates: List of dicts with keys: id, directory, primary_path, files, data
|
||||
"""
|
||||
if not updates:
|
||||
return
|
||||
|
||||
with get_db_curs(commit=True) as curs:
|
||||
for update in updates:
|
||||
# Don't change indexed - reorganization only moves files, content unchanged
|
||||
curs.execute('''
|
||||
UPDATE file_group
|
||||
SET directory = %s,
|
||||
primary_path = %s,
|
||||
files = %s,
|
||||
data = %s
|
||||
WHERE id = %s
|
||||
''', (
|
||||
update['directory'],
|
||||
update['primary_path'],
|
||||
json.dumps(update['files'], default=_json_serial),
|
||||
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
|
||||
update['id'],
|
||||
))
|
||||
|
||||
# Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates.
|
||||
# This is necessary because get_db_curs uses the same connection as get_db_session
|
||||
# during tests, but SQLAlchemy's identity map doesn't know about raw SQL updates.
|
||||
with get_db_session() as session:
|
||||
session.expire_all()
|
||||
|
||||
|
||||
def delete_directory(directory: pathlib.Path, recursive: bool = False):
|
||||
"""Remove a directory, remove it's Directory record.
|
||||
|
||||
|
|
|
|||
|
|
@ -1360,3 +1360,51 @@ async def test_rename_non_primary_file_renames_filegroup(async_client, test_sess
|
|||
assert 'renamed.mp4' in file_names
|
||||
assert 'renamed.srt' in file_names
|
||||
assert len(file_group.files) == 2
|
||||
|
||||
|
||||
|
||||
def test_bulk_update_file_groups_reorganize_handles_datetime(test_session, test_directory):
|
||||
"""_bulk_update_file_groups_reorganize should handle datetime objects in files JSON.
|
||||
|
||||
The files JSON field can contain modification_datetime as a datetime object.
|
||||
The function must serialize these correctly to avoid JSON serialization errors.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from wrolpi.files.lib import _bulk_update_file_groups_reorganize
|
||||
|
||||
# Create a test file and FileGroup
|
||||
test_file = test_directory / 'test_video.mp4'
|
||||
test_file.touch()
|
||||
|
||||
fg = FileGroup()
|
||||
fg.directory = test_directory
|
||||
fg.primary_path = test_file
|
||||
fg.files = [{'path': 'test_video.mp4', 'mimetype': 'video/mp4'}]
|
||||
test_session.add(fg)
|
||||
test_session.commit()
|
||||
fg_id = fg.id
|
||||
|
||||
# Prepare update with datetime object in files (simulating what handle_reorganize does)
|
||||
new_files = [{
|
||||
'path': 'test_video.mp4',
|
||||
'mimetype': 'video/mp4',
|
||||
'modification_datetime': datetime.now(timezone.utc), # datetime object, not string
|
||||
}]
|
||||
|
||||
updates = [{
|
||||
'id': fg_id,
|
||||
'directory': str(test_directory),
|
||||
'primary_path': str(test_file),
|
||||
'files': new_files,
|
||||
}]
|
||||
|
||||
# This should NOT raise "Object of type datetime is not JSON serializable"
|
||||
_bulk_update_file_groups_reorganize(updates)
|
||||
|
||||
# Verify the update was applied
|
||||
test_session.expire_all()
|
||||
fg = test_session.query(FileGroup).filter_by(id=fg_id).one()
|
||||
assert len(fg.files) == 1
|
||||
assert fg.files[0]['path'] == 'test_video.mp4'
|
||||
# modification_datetime should be serialized as ISO string
|
||||
assert 'modification_datetime' in fg.files[0]
|
||||
|
|
|
|||
|
|
@ -302,10 +302,11 @@ async def test_compare_file_groups_ignores_outside_root(test_session, test_direc
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compare_file_groups_cancelable(test_session, test_directory, make_files_structure):
|
||||
"""compare_file_groups can be canceled during long operations."""
|
||||
"""compare_file_groups can be canceled during long operations and reports progress via callback."""
|
||||
# Create many files to ensure scan takes time
|
||||
make_files_structure([f'docs/file{i}.txt' for i in range(100)])
|
||||
|
||||
# Test cancellation
|
||||
task = asyncio.create_task(compare_file_groups(test_directory))
|
||||
await asyncio.sleep(0) # Let task start
|
||||
task.cancel()
|
||||
|
|
@ -313,6 +314,13 @@ async def test_compare_file_groups_cancelable(test_session, test_directory, make
|
|||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
# Test progress callback with small batch size
|
||||
progress_counts = []
|
||||
await compare_file_groups(test_directory, batch_size=10, progress_callback=progress_counts.append)
|
||||
# Should have been called multiple times with increasing counts
|
||||
assert len(progress_counts) >= 1
|
||||
assert progress_counts == sorted(progress_counts) # Counts should be increasing
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_files_empty(test_directory):
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from wrolpi.files.lib import (
|
|||
split_path_stem_and_suffix, _upsert_files, get_unique_files_by_stem, glob_shared_stem,
|
||||
group_files_by_stem, get_primary_file, delete_directory, apply_indexers,
|
||||
_move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE,
|
||||
_bulk_update_file_groups_reorganize,
|
||||
)
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -274,7 +275,11 @@ def _insert_fs_batch(session, batch: list):
|
|||
)
|
||||
|
||||
|
||||
async def compare_file_groups(root: Path = None, batch_size: int = 10000) -> FileComparisonResult:
|
||||
async def compare_file_groups(
|
||||
root: Path = None,
|
||||
batch_size: int = 10000,
|
||||
progress_callback: Callable[[int], None] = None,
|
||||
) -> FileComparisonResult:
|
||||
"""
|
||||
Compare FileGroups in DB to filesystem. Memory efficient for Raspberry Pi.
|
||||
|
||||
|
|
@ -317,6 +322,8 @@ async def compare_file_groups(root: Path = None, batch_size: int = 10000) -> Fil
|
|||
_insert_fs_batch(session, batch)
|
||||
file_count += len(batch)
|
||||
batch = []
|
||||
if progress_callback:
|
||||
progress_callback(file_count)
|
||||
await asyncio.sleep(0) # Cancellation yield point
|
||||
finally:
|
||||
await path_gen.aclose() # Ensure subprocess is killed on cancellation
|
||||
|
|
@ -691,6 +698,8 @@ class FileTaskType(str, Enum):
|
|||
move = auto() # Move files and directories to a new location, file names are not changed.
|
||||
rename = auto() # Rename files, requires deeper updating of FileGroup.files and FileGroup.data.
|
||||
tag = auto() # Add TagFile records for files.
|
||||
reorganize = auto() # Reorganize files with arbitrary source->dest mappings for collection reorganization.
|
||||
batch_reorganize = auto() # Batch reorganize multiple collections sequentially.
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -708,6 +717,15 @@ class FileTask:
|
|||
next_task_type: FileTaskType = None
|
||||
job_id: str = None # Unique ID for tracking completion
|
||||
expand_stems: bool = True # Whether to expand files to their FileGroup stem-mates
|
||||
# For reorganize tasks: list of (source_path, dest_path) tuples
|
||||
move_mappings: list[tuple[pathlib.Path, pathlib.Path]] = None
|
||||
collection_id: int = None # Collection being reorganized
|
||||
collection_kind: str = None # 'channel' or 'domain' for reorganize tasks
|
||||
# For batch_reorganize tasks
|
||||
collection_ids: list[int] = None # List of collection IDs to process
|
||||
batch_kind: str = None # 'channel' or 'domain'
|
||||
# For deferred file_format update (only updated after successful completion)
|
||||
pending_file_format: str = None
|
||||
|
||||
|
||||
class FileWorker:
|
||||
|
|
@ -844,6 +862,75 @@ class FileWorker:
|
|||
self.public_queue.put_nowait(task)
|
||||
return job_id
|
||||
|
||||
def queue_reorganize(
|
||||
self,
|
||||
move_mappings: list[tuple[pathlib.Path, pathlib.Path]],
|
||||
collection_id: int = None,
|
||||
collection_kind: str = None,
|
||||
pending_file_format: str = None,
|
||||
) -> str:
|
||||
"""Queue a reorganize task for collection file reorganization.
|
||||
|
||||
Unlike queue_move which moves all files to a single destination,
|
||||
queue_reorganize handles arbitrary source->destination mappings
|
||||
where each file can move to a different location.
|
||||
|
||||
Args:
|
||||
move_mappings: List of (source_path, dest_path) tuples
|
||||
collection_id: Optional collection ID being reorganized (for status tracking)
|
||||
collection_kind: 'channel' or 'domain' (for status tracking / navbar navigation)
|
||||
pending_file_format: File format to set on collection after successful completion
|
||||
(deferred update enables retry on failure)
|
||||
|
||||
Returns:
|
||||
Job ID for tracking completion
|
||||
"""
|
||||
import uuid
|
||||
job_id = f'reorganize-{uuid.uuid4().hex[:8]}'
|
||||
# Paths list is empty for reorganize tasks - we use move_mappings instead
|
||||
task = FileTask(
|
||||
FileTaskType.reorganize,
|
||||
paths=[],
|
||||
move_mappings=move_mappings,
|
||||
collection_id=collection_id,
|
||||
collection_kind=collection_kind,
|
||||
job_id=job_id,
|
||||
pending_file_format=pending_file_format,
|
||||
)
|
||||
self._set_job_status(job_id, 'pending')
|
||||
self.public_queue.put_nowait(task)
|
||||
return job_id
|
||||
|
||||
def queue_batch_reorganize(
|
||||
self,
|
||||
collection_ids: list[int],
|
||||
kind: str,
|
||||
) -> str:
|
||||
"""Queue a batch reorganize task for multiple collections.
|
||||
|
||||
Processes collections sequentially, stopping on first failure.
|
||||
Tracks overall progress and per-collection progress.
|
||||
|
||||
Args:
|
||||
collection_ids: List of collection IDs to reorganize
|
||||
kind: 'channel' or 'domain'
|
||||
|
||||
Returns:
|
||||
Batch job ID for tracking completion
|
||||
"""
|
||||
import uuid
|
||||
job_id = f'batch-reorganize-{uuid.uuid4().hex[:8]}'
|
||||
task = FileTask(
|
||||
FileTaskType.batch_reorganize,
|
||||
paths=[],
|
||||
collection_ids=collection_ids,
|
||||
batch_kind=kind,
|
||||
job_id=job_id,
|
||||
)
|
||||
self._set_job_status(job_id, 'pending')
|
||||
self.public_queue.put_nowait(task)
|
||||
return job_id
|
||||
|
||||
async def refresh_sync(self, paths: list[pathlib.Path]):
|
||||
"""Synchronously refresh specific files. Use sparingly - prefer queue_refresh."""
|
||||
if not paths:
|
||||
|
|
@ -864,17 +951,32 @@ class FileWorker:
|
|||
try:
|
||||
task: FileTask = self.private_queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
# No file tasks to perform. Sleep to catch cancel.
|
||||
# No tasks - clear flag if set and both queues are empty
|
||||
if flags.file_worker_busy.is_set() and self.public_queue.qsize() == 0:
|
||||
flags.file_worker_busy.clear()
|
||||
await asyncio.sleep(0)
|
||||
return
|
||||
|
||||
match task.task_type:
|
||||
case FileTaskType.count:
|
||||
await self.handle_count(task)
|
||||
case FileTaskType.refresh:
|
||||
await self.handle_refresh(task)
|
||||
case FileTaskType.move:
|
||||
await self.handle_move(task)
|
||||
# Set flag before processing any task (if not already set)
|
||||
if not flags.file_worker_busy.is_set():
|
||||
flags.file_worker_busy.set()
|
||||
|
||||
try:
|
||||
match task.task_type:
|
||||
case FileTaskType.count:
|
||||
await self.handle_count(task)
|
||||
case FileTaskType.refresh:
|
||||
await self.handle_refresh(task)
|
||||
case FileTaskType.move:
|
||||
await self.handle_move(task)
|
||||
case FileTaskType.reorganize:
|
||||
await self.handle_reorganize(task)
|
||||
case FileTaskType.batch_reorganize:
|
||||
await self.handle_batch_reorganize(task)
|
||||
finally:
|
||||
# Clear flag if both queues are now empty
|
||||
if self.private_queue.qsize() == 0 and self.public_queue.qsize() == 0:
|
||||
flags.file_worker_busy.clear()
|
||||
|
||||
async def handle_count(self, task: FileTask):
|
||||
"""Count files in the task's paths and chain to next task if specified."""
|
||||
|
|
@ -1006,8 +1108,12 @@ class FileWorker:
|
|||
)
|
||||
logger.info(f'Comparing {task.count} files in {len(dir_paths)} directories')
|
||||
|
||||
def on_compare_progress(count: int):
|
||||
percent = int((count / task.count) * 100) if task.count > 0 else 0
|
||||
self.update_status(operation_processed=count, operation_percent=percent)
|
||||
|
||||
root = dir_paths[0] if len(dir_paths) == 1 else None
|
||||
dir_result = await compare_file_groups(root=root)
|
||||
dir_result = await compare_file_groups(root=root, progress_callback=on_compare_progress)
|
||||
if is_global_refresh:
|
||||
Events.send_global_refresh_discovery_completed()
|
||||
|
||||
|
|
@ -1661,15 +1767,23 @@ class FileWorker:
|
|||
"""
|
||||
from wrolpi.tags import save_tags_config
|
||||
|
||||
self.update_status(status='modeling')
|
||||
def on_modeling_progress(processed: int, total: int):
|
||||
percent = int((processed / total) * 100) if total > 0 else 0
|
||||
self.update_status(operation_total=total, operation_processed=processed, operation_percent=percent)
|
||||
|
||||
self.update_status(status='modeling', operation_total=0, operation_processed=0, operation_percent=0)
|
||||
with flags.file_worker_modeling:
|
||||
await apply_modelers()
|
||||
await apply_modelers(progress_callback=on_modeling_progress)
|
||||
if is_global_refresh:
|
||||
Events.send_global_refresh_modeling_completed()
|
||||
|
||||
self.update_status(status='indexing')
|
||||
def on_indexing_progress(processed: int, total: int):
|
||||
percent = int((processed / total) * 100) if total > 0 else 0
|
||||
self.update_status(operation_total=total, operation_processed=processed, operation_percent=percent)
|
||||
|
||||
self.update_status(status='indexing', operation_total=0, operation_processed=0, operation_percent=0)
|
||||
with flags.file_worker_indexing:
|
||||
await apply_indexers()
|
||||
await apply_indexers(progress_callback=on_indexing_progress)
|
||||
if is_global_refresh:
|
||||
Events.send_global_refresh_indexing_completed()
|
||||
|
||||
|
|
@ -1763,27 +1877,26 @@ class FileWorker:
|
|||
plan = {}
|
||||
|
||||
try:
|
||||
with flags.file_worker_busy:
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Build the move plan using bulk SQL operations
|
||||
plan, old_directories = await build_move_plan_bulk(sources, destination)
|
||||
# Build the move plan using bulk SQL operations
|
||||
plan, old_directories = await build_move_plan_bulk(sources, destination)
|
||||
|
||||
# Update status with total
|
||||
self.update_status(
|
||||
status='moving',
|
||||
operation_total=len(plan),
|
||||
)
|
||||
# Update status with total
|
||||
self.update_status(
|
||||
status='moving',
|
||||
operation_total=len(plan),
|
||||
)
|
||||
|
||||
# Execute the plan in chunks
|
||||
with flags.file_worker_discovery:
|
||||
with get_db_session(commit=True) as session:
|
||||
await self._execute_move_chunks(
|
||||
plan, session, created_directories, revert_plan
|
||||
)
|
||||
# Execute the plan in chunks
|
||||
with flags.file_worker_discovery:
|
||||
with get_db_session(commit=True) as session:
|
||||
await self._execute_move_chunks(
|
||||
plan, session, created_directories, revert_plan
|
||||
)
|
||||
|
||||
self._cleanup_old_directories(sources, old_directories)
|
||||
await self._apply_post_processing()
|
||||
self._cleanup_old_directories(sources, old_directories)
|
||||
await self._apply_post_processing()
|
||||
|
||||
logger.info(f'Move completed: {len(plan)} items moved to {destination}')
|
||||
Events.send_file_move_completed(f'Moved {len(sources)} items to {destination}')
|
||||
|
|
@ -1800,6 +1913,499 @@ class FileWorker:
|
|||
finally:
|
||||
self.reset_status()
|
||||
|
||||
async def handle_reorganize(self, task: FileTask):
|
||||
"""Handle reorganize task for collection file reorganization.
|
||||
|
||||
Unlike handle_move which moves all files to a single destination,
|
||||
handle_reorganize handles arbitrary source->destination mappings
|
||||
where each file (or FileGroup) can move to a different location.
|
||||
|
||||
This is used when a user changes their file_name_format configuration
|
||||
and wants to reorganize existing files to match the new format.
|
||||
|
||||
Optimized for bulk operations:
|
||||
- Single query to pre-fetch all FileGroups
|
||||
- Uses FileGroup.files instead of filesystem globs
|
||||
- Dict lookup for O(n) path matching
|
||||
- Batched database updates
|
||||
"""
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
move_mappings = task.move_mappings
|
||||
|
||||
if not move_mappings:
|
||||
logger.info('Reorganize task has no move mappings, nothing to do')
|
||||
self._complete_job(task.job_id)
|
||||
self.reset_status()
|
||||
return
|
||||
|
||||
total_moves = len(move_mappings)
|
||||
logger.info(f'Starting reorganize task with {total_moves} file moves')
|
||||
|
||||
# Transition job from 'pending' to 'running' so progress can be tracked
|
||||
self._set_job_status(task.job_id, 'running')
|
||||
|
||||
self.update_status(
|
||||
status='reorganizing',
|
||||
task_type='reorganize',
|
||||
paths=[],
|
||||
destination=None,
|
||||
operation_total=total_moves,
|
||||
operation_processed=0,
|
||||
operation_percent=0,
|
||||
error=None,
|
||||
collection_id=task.collection_id,
|
||||
collection_kind=task.collection_kind,
|
||||
)
|
||||
|
||||
# Track successful moves for rollback
|
||||
completed_moves: list[tuple[pathlib.Path, pathlib.Path]] = []
|
||||
# Track created directories for cleanup
|
||||
created_directories: set[pathlib.Path] = set()
|
||||
# Track source directories to clean up after moves
|
||||
source_directories: set[pathlib.Path] = set()
|
||||
|
||||
try:
|
||||
with flags.file_worker_discovery:
|
||||
# Phase 1: Bulk pre-fetch all FileGroups
|
||||
source_paths = [str(s) for s, d in move_mappings]
|
||||
with get_db_session() as session:
|
||||
file_groups = session.query(FileGroup).filter(
|
||||
FileGroup.primary_path.in_(source_paths)
|
||||
).all()
|
||||
# Build lookup dict and detach from session
|
||||
# Note: fg.primary_path is a pathlib.Path (from MediaPathType), but
|
||||
# source_paths are strings, so we use str() for the dictionary keys
|
||||
fg_by_path = {}
|
||||
for fg in file_groups:
|
||||
fg_by_path[str(fg.primary_path)] = {
|
||||
'id': fg.id,
|
||||
'directory': pathlib.Path(fg.directory),
|
||||
'files': list(fg.files), # Copy the list
|
||||
'data': dict(fg.data) if fg.data else None,
|
||||
}
|
||||
|
||||
processed = 0
|
||||
pending_updates = []
|
||||
|
||||
for source_path, dest_path in move_mappings:
|
||||
source_path = pathlib.Path(source_path)
|
||||
dest_path = pathlib.Path(dest_path)
|
||||
|
||||
source_exists = source_path.exists()
|
||||
dest_exists = dest_path.exists()
|
||||
|
||||
# Look up the pre-fetched FileGroup
|
||||
fg_info = fg_by_path.get(str(source_path))
|
||||
if not fg_info:
|
||||
logger.debug(f'FileGroup not found by primary_path: {source_path}')
|
||||
processed += 1
|
||||
continue
|
||||
|
||||
if source_exists and dest_exists:
|
||||
# Conflict: both exist - skip with warning
|
||||
logger.warning(f'Reorganize: both source and destination exist: {source_path} -> {dest_path}')
|
||||
processed += 1
|
||||
continue
|
||||
|
||||
if not source_exists and not dest_exists:
|
||||
# Neither exists - skip with warning
|
||||
logger.warning(f'Reorganize: neither source nor destination exist: {source_path}')
|
||||
processed += 1
|
||||
continue
|
||||
|
||||
if not source_exists and dest_exists:
|
||||
# Recovery case: file already moved, just update DB to match destination
|
||||
logger.info(f'Reorganize recovery: updating DB for already-moved file: {dest_path}')
|
||||
|
||||
dest_stem, _ = split_path_stem_and_suffix(dest_path)
|
||||
|
||||
# Find existing files at destination that match the FileGroup's file extensions
|
||||
moved_files = []
|
||||
for file_info in fg_info['files']:
|
||||
_, suffix = split_path_stem_and_suffix(pathlib.Path(file_info['path']))
|
||||
dest_file = dest_path.parent / f'{dest_stem}{suffix}'
|
||||
if dest_file.exists():
|
||||
# Map original source path to destination for DB update
|
||||
src_file = fg_info['directory'] / file_info['path']
|
||||
moved_files.append((src_file, dest_file))
|
||||
|
||||
if moved_files:
|
||||
# Build DB update using existing destination files
|
||||
moved_lookup = {src: dst for src, dst in moved_files}
|
||||
new_files = []
|
||||
for file_info in fg_info['files']:
|
||||
old_file_path = pathlib.Path(file_info['path'])
|
||||
if not old_file_path.is_absolute():
|
||||
old_file_path = fg_info['directory'] / old_file_path
|
||||
if old_file_path in moved_lookup:
|
||||
file_info = dict(file_info)
|
||||
file_info['path'] = moved_lookup[old_file_path].name
|
||||
new_files.append(file_info)
|
||||
|
||||
new_data = fg_info['data']
|
||||
if new_data:
|
||||
new_data = dict(new_data)
|
||||
filename_lookup = {src.name: dst.name for src, dst in moved_files}
|
||||
for key, value in new_data.items():
|
||||
if isinstance(value, str) and value in filename_lookup:
|
||||
new_data[key] = filename_lookup[value]
|
||||
elif isinstance(value, list):
|
||||
new_data[key] = [filename_lookup.get(v, v) for v in value]
|
||||
|
||||
pending_updates.append({
|
||||
'id': fg_info['id'],
|
||||
'directory': str(dest_path.parent),
|
||||
'primary_path': str(dest_path),
|
||||
'files': new_files,
|
||||
'data': new_data,
|
||||
})
|
||||
completed_moves.append((source_path, dest_path))
|
||||
|
||||
processed += 1
|
||||
if processed % PROGRESS_UPDATE_INTERVAL == 0:
|
||||
percent = int((processed / total_moves) * 100)
|
||||
self.update_status(operation_processed=processed, operation_percent=percent)
|
||||
continue
|
||||
|
||||
# Normal case: source exists, dest doesn't - move files
|
||||
# Track source directory for cleanup
|
||||
source_directories.add(source_path.parent)
|
||||
|
||||
# Create destination directory if needed
|
||||
if not dest_path.parent.exists():
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
created_directories.add(dest_path.parent)
|
||||
|
||||
# Phase 2: Use FileGroup.files instead of glob_shared_stem()
|
||||
source_stem_files = []
|
||||
for file_info in fg_info['files']:
|
||||
file_path = pathlib.Path(file_info['path'])
|
||||
if not file_path.is_absolute():
|
||||
file_path = fg_info['directory'] / file_path
|
||||
source_stem_files.append(file_path)
|
||||
|
||||
# Move all files in the FileGroup
|
||||
dest_stem, _ = split_path_stem_and_suffix(dest_path)
|
||||
moved_files = []
|
||||
for src_file in source_stem_files:
|
||||
if not src_file.exists():
|
||||
continue
|
||||
# Compute destination for this file based on its suffix
|
||||
_, src_suffix = split_path_stem_and_suffix(src_file)
|
||||
dest_file = dest_path.parent / f'{dest_stem}{src_suffix}'
|
||||
|
||||
try:
|
||||
shutil.move(str(src_file), str(dest_file))
|
||||
moved_files.append((src_file, dest_file))
|
||||
logger.debug(f'Reorganize: moved {src_file} -> {dest_file}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to move {src_file} -> {dest_file}: {e}')
|
||||
raise
|
||||
|
||||
# Prepare FileGroup update
|
||||
if moved_files:
|
||||
# Phase 3: Use dict lookup instead of nested loop
|
||||
moved_lookup = {src: dst for src, dst in moved_files}
|
||||
|
||||
# Update files list with new paths
|
||||
new_files = []
|
||||
for file_info in fg_info['files']:
|
||||
old_file_path = pathlib.Path(file_info['path'])
|
||||
if not old_file_path.is_absolute():
|
||||
old_file_path = fg_info['directory'] / old_file_path
|
||||
|
||||
# Find the corresponding new path using dict lookup
|
||||
if old_file_path in moved_lookup:
|
||||
file_info = dict(file_info)
|
||||
file_info['path'] = moved_lookup[old_file_path].name
|
||||
new_files.append(file_info)
|
||||
|
||||
# Update data dict if it has path references
|
||||
# Data fields store filenames only (no '/'), so use filename-to-filename lookup
|
||||
new_data = fg_info['data']
|
||||
if new_data:
|
||||
new_data = dict(new_data)
|
||||
# Build filename-to-filename lookup (not full paths)
|
||||
filename_lookup = {src.name: dst.name for src, dst in moved_files}
|
||||
for key, value in new_data.items():
|
||||
if isinstance(value, str) and value in filename_lookup:
|
||||
new_data[key] = filename_lookup[value]
|
||||
elif isinstance(value, list):
|
||||
# Handle list fields like caption_paths
|
||||
new_data[key] = [filename_lookup.get(v, v) for v in value]
|
||||
|
||||
# Phase 4: Collect update for batch processing
|
||||
pending_updates.append({
|
||||
'id': fg_info['id'],
|
||||
'directory': str(dest_path.parent),
|
||||
'primary_path': str(dest_path),
|
||||
'files': new_files,
|
||||
'data': new_data,
|
||||
})
|
||||
completed_moves.append((source_path, dest_path))
|
||||
|
||||
# Flush every MOVE_CHUNK_SIZE items
|
||||
if len(pending_updates) >= MOVE_CHUNK_SIZE:
|
||||
_bulk_update_file_groups_reorganize(pending_updates)
|
||||
pending_updates = []
|
||||
|
||||
processed += 1
|
||||
|
||||
# Update progress
|
||||
if processed % PROGRESS_UPDATE_INTERVAL == 0 or processed == total_moves:
|
||||
percent = int((processed / total_moves) * 100)
|
||||
logger.info(f'Reorganize progress: {processed}/{total_moves} files ({percent}%)')
|
||||
self.update_status(
|
||||
operation_processed=processed,
|
||||
operation_percent=percent,
|
||||
)
|
||||
|
||||
# Final flush of pending updates
|
||||
if pending_updates:
|
||||
_bulk_update_file_groups_reorganize(pending_updates)
|
||||
|
||||
# Clean up empty source directories
|
||||
for directory in sorted(source_directories, key=lambda p: len(p.parents), reverse=True):
|
||||
try:
|
||||
if directory.is_dir() and not any(directory.iterdir()):
|
||||
delete_directory(directory)
|
||||
logger.debug(f'Cleaned up empty directory: {directory}')
|
||||
except (OSError, StopIteration):
|
||||
pass
|
||||
|
||||
# Skip post-processing - reorganization only moves files, nothing to re-index
|
||||
|
||||
# Update file_format after successful completion (deferred update for resumability)
|
||||
if task.collection_id and task.pending_file_format:
|
||||
from wrolpi.collections.models import Collection
|
||||
with get_db_session(commit=True) as session:
|
||||
collection = session.query(Collection).filter_by(id=task.collection_id).one_or_none()
|
||||
if collection:
|
||||
collection.file_format = task.pending_file_format
|
||||
logger.info(f'Updated file_format for collection {collection.name}')
|
||||
# Trigger config save
|
||||
if collection.kind == 'domain':
|
||||
from modules.archive.lib import save_domains_config
|
||||
save_domains_config.activate_switch()
|
||||
elif collection.kind == 'channel':
|
||||
from modules.videos.lib import save_channels_config
|
||||
save_channels_config.activate_switch()
|
||||
|
||||
logger.info(f'Reorganize completed: {len(completed_moves)} files moved')
|
||||
Events.send_file_move_completed(f'Reorganized {len(completed_moves)} files')
|
||||
self._complete_job(task.job_id)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.warning('Reorganize task was cancelled')
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f'Reorganize failed: {e}', exc_info=e)
|
||||
self.update_status(status='error', error=str(e))
|
||||
# Note: We don't attempt automatic rollback for reorganize tasks
|
||||
# as the partial state may be complex. User should refresh files.
|
||||
Events.send_file_move_failed(f'Reorganize failed: {e}')
|
||||
finally:
|
||||
self.reset_status()
|
||||
|
||||
async def handle_batch_reorganize(self, task: FileTask):
|
||||
"""Handle batch reorganization of multiple collections.
|
||||
|
||||
Processes collections sequentially:
|
||||
1. For each collection, execute reorganization
|
||||
2. Track overall progress and per-collection progress
|
||||
3. Stop on first failure, report which collection failed
|
||||
|
||||
Progress tracking:
|
||||
- overall_percent = (completed_collections + current_collection_%) / total_collections
|
||||
- batch_status dict tracks completed list, current collection, and failure info
|
||||
"""
|
||||
from wrolpi.collections.reorganize import execute_reorganization
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.events import Events
|
||||
|
||||
collection_ids = task.collection_ids or []
|
||||
kind = task.batch_kind
|
||||
|
||||
if not collection_ids:
|
||||
logger.info('Batch reorganize task has no collections, nothing to do')
|
||||
self._complete_job(task.job_id)
|
||||
self.reset_status()
|
||||
return
|
||||
|
||||
total_collections = len(collection_ids)
|
||||
logger.info(f'Starting batch reorganization of {total_collections} {kind} collections')
|
||||
|
||||
# Transition job from 'pending' to 'running'
|
||||
self._set_job_status(task.job_id, 'running')
|
||||
|
||||
# Initialize batch status tracking
|
||||
batch_status = {
|
||||
'batch_job_id': task.job_id, # Include job ID so frontend can resume polling
|
||||
'total_collections': total_collections,
|
||||
'completed': [],
|
||||
'current_collection': None,
|
||||
'failed_collection': None,
|
||||
'error': None,
|
||||
'overall_percent': 0,
|
||||
}
|
||||
|
||||
self.update_status(
|
||||
status='batch_reorganizing',
|
||||
task_type='batch_reorganize',
|
||||
paths=[],
|
||||
destination=None,
|
||||
operation_total=total_collections,
|
||||
operation_processed=0,
|
||||
operation_percent=0,
|
||||
error=None,
|
||||
batch_status=batch_status,
|
||||
collection_kind=kind,
|
||||
)
|
||||
|
||||
try:
|
||||
for idx, collection_id in enumerate(collection_ids):
|
||||
# Fetch collection info
|
||||
with get_db_session() as session:
|
||||
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
|
||||
if not collection:
|
||||
logger.warning(f'Batch reorganize: collection {collection_id} not found, skipping')
|
||||
continue
|
||||
|
||||
collection_name = collection.name
|
||||
|
||||
# Update current collection in status
|
||||
batch_status['current_collection'] = {
|
||||
'id': collection_id,
|
||||
'name': collection_name,
|
||||
'status': 'running',
|
||||
'total': 0,
|
||||
'completed': 0,
|
||||
'percent': 0,
|
||||
}
|
||||
self.update_status(batch_status=batch_status)
|
||||
|
||||
logger.info(f'Batch reorganize: processing {collection_name} ({idx + 1}/{total_collections})')
|
||||
|
||||
try:
|
||||
# Execute reorganization for this collection
|
||||
# This queues a sub-job that we need to wait for
|
||||
job_id = execute_reorganization(collection_id, session)
|
||||
|
||||
if job_id:
|
||||
# Wait for the reorganization job to complete
|
||||
# Poll for completion while updating batch status
|
||||
await self._wait_for_sub_job_with_progress(
|
||||
job_id, batch_status, collection_id, collection_name, idx, total_collections
|
||||
)
|
||||
|
||||
# Mark this collection as completed
|
||||
batch_status['completed'].append({
|
||||
'id': collection_id,
|
||||
'name': collection_name,
|
||||
'status': 'complete',
|
||||
})
|
||||
|
||||
# Update overall progress
|
||||
completed_count = len(batch_status['completed'])
|
||||
batch_status['overall_percent'] = int((completed_count / total_collections) * 100)
|
||||
batch_status['current_collection'] = None
|
||||
self.update_status(
|
||||
operation_processed=completed_count,
|
||||
operation_percent=batch_status['overall_percent'],
|
||||
batch_status=batch_status,
|
||||
)
|
||||
|
||||
logger.info(f'Batch reorganize: completed {collection_name}')
|
||||
|
||||
except Exception as e:
|
||||
# Collection failed - stop batch processing
|
||||
logger.error(f'Batch reorganize: {collection_name} failed: {e}')
|
||||
batch_status['failed_collection'] = {
|
||||
'id': collection_id,
|
||||
'name': collection_name,
|
||||
}
|
||||
batch_status['error'] = str(e)
|
||||
batch_status['current_collection'] = None
|
||||
self.update_status(
|
||||
status='error',
|
||||
error=str(e),
|
||||
batch_status=batch_status,
|
||||
)
|
||||
self._set_job_status(task.job_id, 'failed')
|
||||
Events.send_file_move_failed(
|
||||
f'Batch reorganization failed on {collection_name}: {e}'
|
||||
)
|
||||
return
|
||||
|
||||
# All collections completed successfully
|
||||
logger.info(f'Batch reorganization completed: {len(batch_status["completed"])} collections processed')
|
||||
Events.send_file_move_completed(
|
||||
f'Batch reorganization completed: {len(batch_status["completed"])} {kind}s reorganized'
|
||||
)
|
||||
self._complete_job(task.job_id)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.warning('Batch reorganize task was cancelled')
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f'Batch reorganize failed: {e}', exc_info=e)
|
||||
batch_status['error'] = str(e)
|
||||
self.update_status(status='error', error=str(e), batch_status=batch_status)
|
||||
self._set_job_status(task.job_id, 'failed')
|
||||
Events.send_file_move_failed(f'Batch reorganization failed: {e}')
|
||||
finally:
|
||||
self.reset_status()
|
||||
|
||||
async def _wait_for_sub_job_with_progress(
|
||||
self,
|
||||
job_id: str,
|
||||
batch_status: dict,
|
||||
collection_id: int,
|
||||
collection_name: str,
|
||||
collection_idx: int,
|
||||
total_collections: int,
|
||||
):
|
||||
"""Wait for a sub-job to complete while updating batch progress.
|
||||
|
||||
This is used by handle_batch_reorganize to wait for each collection's
|
||||
reorganization job while keeping the batch status updated.
|
||||
"""
|
||||
import time
|
||||
start = time.time()
|
||||
timeout = 3600 # 1 hour max per collection
|
||||
|
||||
while time.time() - start < timeout:
|
||||
status = self.get_job_status(job_id)
|
||||
if status == 'complete':
|
||||
return
|
||||
|
||||
# Transfer and process queue FIRST to ensure status is updated
|
||||
self.transfer_queue()
|
||||
await self.process_queue()
|
||||
|
||||
# Update current collection progress based on worker status (now fresh)
|
||||
worker_status = self.status
|
||||
current = batch_status.get('current_collection', {})
|
||||
if current:
|
||||
current['total'] = worker_status.get('operation_total', 0)
|
||||
current['completed'] = worker_status.get('operation_processed', 0)
|
||||
current['percent'] = worker_status.get('operation_percent', 0)
|
||||
batch_status['current_collection'] = current
|
||||
|
||||
# Calculate overall percent including current progress
|
||||
completed_count = len(batch_status.get('completed', []))
|
||||
current_progress = current['percent'] / 100
|
||||
batch_status['overall_percent'] = int(
|
||||
((completed_count + current_progress) / total_collections) * 100
|
||||
)
|
||||
self.update_status(batch_status=batch_status)
|
||||
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
raise TimeoutError(f'Collection {collection_name} reorganization timed out')
|
||||
|
||||
|
||||
# All processes create a FileWorker, but only one receives the signals and actually does work.
|
||||
file_worker = FileWorker()
|
||||
|
|
|
|||
|
|
@ -18,7 +18,11 @@ class MediaPathType(types.TypeDecorator): # noqa
|
|||
raise ValueError('MediaPath cannot be empty')
|
||||
|
||||
if isinstance(value, pathlib.Path):
|
||||
value = str(value.absolute())
|
||||
if not value.is_absolute():
|
||||
# Resolve relative paths against media directory, not cwd
|
||||
from wrolpi.common import get_media_directory
|
||||
value = get_media_directory() / value
|
||||
value = str(value)
|
||||
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f'Invalid MediaPath type ({type(value)}): {value}')
|
||||
|
|
|
|||
|
|
@ -1015,29 +1015,60 @@ async def test_config_valid(async_client, test_wrolpi_config):
|
|||
@pytest.mark.parametrize('name,expected_name', [
|
||||
('foo', 'foo'),
|
||||
(pathlib.Path('foo'), pathlib.Path('foo')),
|
||||
# Short filenames with parents (no trimming needed) should preserve parent directories.
|
||||
('some/parent/short.txt', 'some/parent/short.txt'),
|
||||
('/absolute/parent/short.txt', '/absolute/parent/short.txt'),
|
||||
(pathlib.Path('/some/parent/short.txt'), pathlib.Path('/some/parent/short.txt')),
|
||||
# Long ASCII filenames are trimmed to 200 BYTES (not characters)
|
||||
(
|
||||
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.txt',
|
||||
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt'
|
||||
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt'
|
||||
),
|
||||
(
|
||||
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json',
|
||||
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
|
||||
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
|
||||
),
|
||||
(
|
||||
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json',
|
||||
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
|
||||
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
|
||||
),
|
||||
(
|
||||
pathlib.Path(
|
||||
'/some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.readability.json'),
|
||||
pathlib.Path(
|
||||
'/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json')
|
||||
'/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json')
|
||||
),
|
||||
])
|
||||
def test_trim_file_name(name, expected_name):
|
||||
assert common.trim_file_name(name) == expected_name
|
||||
|
||||
|
||||
def test_trim_file_name_utf8_bytes():
|
||||
"""UTF-8 multi-byte characters must be trimmed by BYTE length, not character length.
|
||||
|
||||
Linux ext4/xfs filesystems have a 255 BYTE limit, not a 255 character limit.
|
||||
Hindi/Devanagari characters are 3 bytes each in UTF-8, so a 121-character filename
|
||||
can be 263 bytes, exceeding the limit.
|
||||
|
||||
This test reproduces the bug from: https://decay.local reorganization failure
|
||||
"""
|
||||
# This filename has 133 characters but 291 bytes - exceeds 255 byte limit
|
||||
hindi_filename = '2022-04-13-19-40-51_भारतीय संस्कृति और परंपराएं बहुत समृद्ध हैं हमारे देश में विविधता में एकता है यह हमारी विरासत है.readability.html'
|
||||
assert len(hindi_filename) == 133 # Characters
|
||||
assert len(hindi_filename.encode('utf-8')) == 291 # Bytes - exceeds 255!
|
||||
|
||||
result = common.trim_file_name(hindi_filename)
|
||||
|
||||
# Result must fit within filesystem byte limit (200 bytes leaves room for suffix variations)
|
||||
max_bytes = 200
|
||||
result_bytes = len(result.encode('utf-8'))
|
||||
assert result_bytes <= max_bytes, \
|
||||
f'Trimmed filename is {result_bytes} bytes, exceeds {max_bytes} byte limit'
|
||||
|
||||
# Must preserve the suffix
|
||||
assert result.endswith('.readability.html'), f'Suffix not preserved: {result}'
|
||||
|
||||
|
||||
async def test_cached_multiprocessing_result(async_client):
|
||||
call_count = 0
|
||||
|
||||
|
|
|
|||
85
wrolpi/test_media_path.py
Normal file
85
wrolpi/test_media_path.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
"""
|
||||
Tests for MediaPathType to ensure paths are resolved correctly.
|
||||
"""
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.common import get_media_directory
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
||||
|
||||
def test_media_path_type_relative_path_resolved_against_media_directory(test_session, test_directory):
|
||||
"""MediaPathType should resolve relative paths against media directory, not cwd."""
|
||||
relative_path = pathlib.Path('archive/test.com')
|
||||
collection = Collection(name='test.com', kind='domain', directory=relative_path)
|
||||
test_session.add(collection)
|
||||
test_session.commit()
|
||||
test_session.expire(collection)
|
||||
|
||||
assert collection.directory.is_absolute()
|
||||
assert collection.directory == test_directory / 'archive/test.com'
|
||||
|
||||
|
||||
def test_media_path_type_process_bind_param_relative_path(test_directory):
|
||||
"""process_bind_param should resolve relative paths against media directory."""
|
||||
media_path_type = MediaPathType()
|
||||
relative_path = pathlib.Path('archive/test.com')
|
||||
result = media_path_type.process_bind_param(relative_path, dialect=None)
|
||||
|
||||
assert result == str(test_directory / 'archive/test.com')
|
||||
assert not result.startswith(os.getcwd()) # NOT resolved against cwd
|
||||
|
||||
|
||||
def test_collection_get_or_set_directory_database_roundtrip(test_session, test_directory, test_wrolpi_config):
|
||||
"""get_or_set_directory should store and retrieve correct paths."""
|
||||
from unittest import mock
|
||||
|
||||
collection = Collection(name='example.com', kind='domain')
|
||||
test_session.add(collection)
|
||||
test_session.commit()
|
||||
|
||||
# Mock switch activation since shared context isn't set up in unit tests
|
||||
with mock.patch('modules.archive.lib.save_domains_config.activate_switch'):
|
||||
result1 = collection.get_or_set_directory(test_session)
|
||||
test_session.expire(collection)
|
||||
result2 = collection.get_or_set_directory(test_session)
|
||||
|
||||
assert result1 == result2
|
||||
assert str(test_directory) in str(result2)
|
||||
assert '/opt/wrolpi' not in str(collection.directory)
|
||||
|
||||
|
||||
def test_media_path_type_absolute_path_unchanged(test_session, test_directory):
|
||||
"""Absolute paths under media directory should be stored unchanged."""
|
||||
absolute_path = test_directory / 'archive/test.com'
|
||||
collection = Collection(name='test.com', kind='domain', directory=absolute_path)
|
||||
test_session.add(collection)
|
||||
test_session.commit()
|
||||
test_session.expire(collection)
|
||||
|
||||
assert collection.directory == absolute_path
|
||||
|
||||
|
||||
def test_media_path_type_none_unchanged(test_directory):
|
||||
"""None should pass through unchanged."""
|
||||
media_path_type = MediaPathType()
|
||||
result = media_path_type.process_bind_param(None, dialect=None)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_media_path_type_empty_string_raises(test_directory):
|
||||
"""Empty string should raise ValueError."""
|
||||
media_path_type = MediaPathType()
|
||||
with pytest.raises(ValueError, match='MediaPath cannot be empty'):
|
||||
media_path_type.process_bind_param('', dialect=None)
|
||||
|
||||
|
||||
def test_media_path_type_string_path_passed_through(test_directory):
|
||||
"""String paths should be passed through as-is (assumed already absolute)."""
|
||||
media_path_type = MediaPathType()
|
||||
path_str = str(test_directory / 'archive/test.com')
|
||||
result = media_path_type.process_bind_param(path_str, dialect=None)
|
||||
assert result == path_str
|
||||
Loading…
Add table
Add a link
Reference in a new issue