diff --git a/alembic/versions/a7b1c2d3e4f5_collection_item_polymorphic.py b/alembic/versions/a7b1c2d3e4f5_collection_item_polymorphic.py new file mode 100644 index 00000000..ceb50bae --- /dev/null +++ b/alembic/versions/a7b1c2d3e4f5_collection_item_polymorphic.py @@ -0,0 +1,58 @@ +"""Make collection_item polymorphic: file | zim | url items. + +CollectionItem previously only referenced a FileGroup. Playlists (kind='playlist') also need to +order Zim articles (zim_id + zim_entry, mirroring TagZimEntry) and arbitrary URLs (a link the WROLPi +browser can open, e.g. a map location). This adds item_kind plus the zim/url/title columns, relaxes +file_group_id to nullable, and adds per-kind unique constraints and a CHECK that exactly one kind's +columns are populated. Existing rows default to item_kind='file'. + +Revision ID: a7b1c2d3e4f5 +Revises: e6f7a8b9c0d1 +Create Date: 2026-06-08 + +""" +from alembic import op +import sqlalchemy as sa + +revision = 'a7b1c2d3e4f5' +down_revision = 'e6f7a8b9c0d1' +branch_labels = None +depends_on = None + +CHECK_SQL = ( + "(item_kind = 'file' AND file_group_id IS NOT NULL AND zim_id IS NULL AND url IS NULL)" + " OR (item_kind = 'zim' AND zim_id IS NOT NULL AND zim_entry IS NOT NULL" + " AND file_group_id IS NULL AND url IS NULL)" + " OR (item_kind = 'url' AND url IS NOT NULL AND file_group_id IS NULL AND zim_id IS NULL)" +) + + +def upgrade(): + op.add_column('collection_item', + sa.Column('item_kind', sa.String(), nullable=False, server_default='file')) + op.add_column('collection_item', sa.Column('zim_id', sa.Integer(), nullable=True)) + op.add_column('collection_item', sa.Column('zim_entry', sa.Text(), nullable=True)) + op.add_column('collection_item', sa.Column('url', sa.Text(), nullable=True)) + op.add_column('collection_item', sa.Column('title', sa.Text(), nullable=True)) + # `file` is no longer the only kind, so a row need not have a FileGroup. + op.alter_column('collection_item', 'file_group_id', existing_type=sa.Integer(), nullable=True) + op.create_foreign_key('collection_item_zim_id_fkey', 'collection_item', 'zim', + ['zim_id'], ['id'], ondelete='CASCADE') + op.create_unique_constraint('uq_collection_zim_entry', 'collection_item', + ['collection_id', 'zim_id', 'zim_entry']) + op.create_unique_constraint('uq_collection_url', 'collection_item', ['collection_id', 'url']) + op.create_check_constraint('ck_collection_item_kind', 'collection_item', CHECK_SQL) + + +def downgrade(): + op.drop_constraint('ck_collection_item_kind', 'collection_item', type_='check') + op.drop_constraint('uq_collection_url', 'collection_item', type_='unique') + op.drop_constraint('uq_collection_zim_entry', 'collection_item', type_='unique') + op.drop_constraint('collection_item_zim_id_fkey', 'collection_item', type_='foreignkey') + # Restore NOT NULL (valid only if no non-file rows remain). + op.alter_column('collection_item', 'file_group_id', existing_type=sa.Integer(), nullable=False) + op.drop_column('collection_item', 'title') + op.drop_column('collection_item', 'url') + op.drop_column('collection_item', 'zim_entry') + op.drop_column('collection_item', 'zim_id') + op.drop_column('collection_item', 'item_kind') diff --git a/app/src/App.js b/app/src/App.js index 93be4c00..4b1e0b9c 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -23,6 +23,7 @@ import {SemanticToastContainer} from "react-semantic-toasts-2"; import {FilePreviewProvider} from "./components/FilePreview"; import {TagsProvider} from "./Tags"; import {ZimRoute} from "./components/Zim"; +import {PlaylistsRoute} from "./components/Playlists"; import {DocsRoute} from "./components/Docs"; import ErrorBoundary from "./components/ErrorBoundary"; import {KeyboardShortcutsProvider} from "./components/KeyboardShortcutsProvider"; @@ -104,6 +105,7 @@ const router = createBrowserRouter(createRoutesFromElements(}/> }/> }/> + }/> }/> )); diff --git a/app/src/api.js b/app/src/api.js index 07c0264a..5665836f 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -2264,3 +2264,101 @@ export async function searchDocs(offset, limit, searchStr, order, tagNames, auth throw Error(message); } } + + +export async function fetchPlaylists() { + const response = await apiGet(`${COLLECTIONS_API}?kind=playlist`); + if (response.ok) { + return (await response.json())['collections']; + } + const message = await getErrorMessage(response, 'Failed to fetch playlists.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); +} + +export async function getPlaylist(playlistId) { + const response = await apiGet(`${COLLECTIONS_API}/${playlistId}`); + if (response.ok) { + return (await response.json())['collection']; + } + const message = await getErrorMessage(response, 'Failed to fetch playlist.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); +} + +export async function createPlaylist(name, description) { + // Trailing slash avoids Sanic's 302 redirect on the root POST route (which would drop the body). + const response = await apiPost(`${COLLECTIONS_API}/`, {name, description, kind: 'playlist'}); + if (response.ok) { + return (await response.json())['collection']; + } + const message = await getErrorMessage(response, 'Failed to create playlist.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); +} + +export async function deletePlaylist(playlistId) { + const response = await apiDelete(`${COLLECTIONS_API}/${playlistId}`); + if (!response.ok) { + const message = await getErrorMessage(response, 'Failed to delete playlist.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); + } +} + +export async function addPlaylistItem(playlistId, item) { + // item: {item_kind, url?, title?, file_group_id?, zim_id?, zim_entry?, position?} + const response = await apiPost(`${COLLECTIONS_API}/${playlistId}/items`, item); + if (response.ok) { + return (await response.json())['item']; + } + const message = await getErrorMessage(response, 'Failed to add item.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); +} + +export async function removePlaylistItem(playlistId, itemId) { + const response = await apiDelete(`${COLLECTIONS_API}/${playlistId}/items/${itemId}`); + if (!response.ok) { + const message = await getErrorMessage(response, 'Failed to remove item.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); + } +} + +export async function reorderPlaylistItems(playlistId, itemIds) { + const response = await apiPut(`${COLLECTIONS_API}/${playlistId}/items/order`, {item_ids: itemIds}); + if (response.ok) { + return (await response.json())['collection']; + } + const message = await getErrorMessage(response, 'Failed to reorder items.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); +} + +// Update a playlist's name, description, and/or custom directory ('' clears the directory). +export async function updatePlaylist(playlistId, {name, description, directory}) { + const response = await apiPut(`${COLLECTIONS_API}/${playlistId}`, {name, description, directory}); + if (response.ok) { + return (await response.json())['collection']; + } + const message = await getErrorMessage(response, 'Failed to update playlist.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); +} + +// Set (tagName) or clear (tagName='') a playlist's single tag. When `directory` is given, the +// playlist also moves there (the managed tag location is normalized back to auto-managed). +export async function setPlaylistTag(playlistId, tagName, directory) { + const body = {tag_name: tagName || ''}; + if (directory !== null && directory !== undefined) { + body.directory = directory; + } + const response = await apiPut(`${COLLECTIONS_API}/${playlistId}`, body); + if (response.ok) { + return (await response.json())['collection']; + } + const message = await getErrorMessage(response, 'Failed to update playlist tag.'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); + throw Error(message); +} diff --git a/app/src/components/AddToPlaylist.js b/app/src/components/AddToPlaylist.js new file mode 100644 index 00000000..c80b244d --- /dev/null +++ b/app/src/components/AddToPlaylist.js @@ -0,0 +1,132 @@ +import React, {useEffect, useState} from "react"; +import {Input, Message} from "semantic-ui-react"; +import {toast} from "react-semantic-toasts-2"; + +import {Button, Form, List, Loader, Modal} from "./Theme"; +import {addPlaylistItem, createPlaylist, fetchPlaylists} from "../api"; + + +/** + * Modal that adds one or more items to a playlist — an existing one or a newly-created one. + * + * `items` is a list of item payloads, each ready to POST to .../items, e.g. + * {item_kind: 'file', file_group_id: 123} + * {item_kind: 'zim', zim_id: 1, zim_entry: 'A/Fire', title: 'Fire'} + * {item_kind: 'url', url: '/map?...', title: '...'} + */ +export function AddToPlaylistModal({open, onClose, items = [], onComplete}) { + const [playlists, setPlaylists] = useState(null); // null=loading, []/[...]=ok, undefined=error + const [selected, setSelected] = useState(null); + const [newName, setNewName] = useState(''); + const [adding, setAdding] = useState(false); + + useEffect(() => { + if (!open) return; + setSelected(null); + setNewName(''); + setPlaylists(null); + fetchPlaylists().then(setPlaylists).catch(() => setPlaylists(undefined)); + }, [open]); + + const addAll = async (playlistId, playlistName) => { + setAdding(true); + try { + for (const payload of items) { + await addPlaylistItem(playlistId, payload); + } + const n = items.length; + toast({ + type: 'success', title: 'Added to playlist', + description: `${n} item${n === 1 ? '' : 's'} added to "${playlistName}"`, time: 3000, + }); + if (onComplete) onComplete(); + onClose(); + } catch (e) { + // Error toast already shown by the API client. + } finally { + setAdding(false); + } + }; + + const handleAddExisting = async () => { + const playlist = (playlists || []).find(p => p.id === selected); + if (playlist) await addAll(playlist.id, playlist.name); + }; + + const handleCreateAndAdd = async () => { + const name = newName.trim(); + if (!name) return; + try { + const playlist = await createPlaylist(name); + if (playlist && playlist.id) await addAll(playlist.id, playlist.name); + } catch (e) { + } + }; + + const creating = !!newName.trim(); + + return + Add to Playlist + + {playlists === undefined && Could not load playlists} + {playlists === null && } + {Array.isArray(playlists) && playlists.length > 0 && + + {playlists.map(p => setSelected(p.id)}> + + {p.name} + )} + } + {Array.isArray(playlists) && playlists.length === 0 && +

No playlists yet — create one below.

} +
+ + + setNewName(value)}/> + +
+
+ + + {creating + ? + : } + +
; +} + + +/** + * A button that opens AddToPlaylistModal for one of: + * - one FileGroup (fileGroupId) or many (fileGroupIds), + * - a Zim article (zim={zimId, entry, title}), or + * - a URL (url={url, title}) — link to anything the WROLPi browser can open (e.g. a map location). + * Extra props are forwarded to the Button (color, size, disabled, icon, content, ...). + */ +export function AddToPlaylistButton({fileGroupId, fileGroupIds, zim, url, content = 'Add to Playlist', onComplete, ...buttonProps}) { + const [open, setOpen] = useState(false); + + let items; + if (url && url.url) { + items = [{item_kind: 'url', url: url.url, title: url.title || null}]; + } else if (zim && zim.zimId != null && zim.entry) { + items = [{item_kind: 'zim', zim_id: zim.zimId, zim_entry: zim.entry, title: zim.title || null}]; + } else { + const ids = fileGroupIds || (fileGroupId != null ? [fileGroupId] : []); + items = ids.map(id => ({item_kind: 'file', file_group_id: id})); + } + + // An empty `content` makes a compact icon-only button (omit content entirely so Semantic does + // not reserve label padding); otherwise render the labeled "Add to Playlist" button. + const contentProps = content ? {content} : {}; + + return <> + ; diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 5774f00a..2e16fafd 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -191,7 +191,7 @@ export function ChannelEditPage() { type="button" size='small' onClick={() => setTagEditModalOpen(true)} - color='green' + color='violet' style={{marginTop: '1em'}} >Tag; diff --git a/app/src/components/Docs.js b/app/src/components/Docs.js index 578c6e3c..21bd6e75 100644 --- a/app/src/components/Docs.js +++ b/app/src/components/Docs.js @@ -27,6 +27,7 @@ import {useAuthors, useDoc, useOneQuery, useSearchDocs, useSubjects} from "../ho import {TagsSelector} from "../Tags"; import {toast} from "react-semantic-toasts-2"; import {CollectionTable} from "./collections/CollectionTable"; +import {AddToPlaylistButton} from "./AddToPlaylist"; import {CbzViewer} from "./CbzViewer"; import _ from "lodash"; @@ -438,6 +439,7 @@ function DocPage() { onClick={() => handleDelete(false)} obeyWROLMode={true} >Delete + ; return <> diff --git a/app/src/components/FilePreview.js b/app/src/components/FilePreview.js index e3ee42df..61502d8f 100644 --- a/app/src/components/FilePreview.js +++ b/app/src/components/FilePreview.js @@ -11,6 +11,7 @@ import {Button, Modal} from "./Theme"; import {toast} from "react-semantic-toasts-2"; import {useOneQuery} from "../hooks/customHooks"; import {ShareButton} from "./Share"; +import {AddToPlaylistButton} from "./AddToPlaylist"; import {pathDirectory} from "./FileBrowser"; import {InlineErrorBoundary} from "./ErrorBoundary"; import {useLocation} from "react-router"; @@ -496,6 +497,9 @@ export function FilePreviewProvider({children}) { {downloadButton} {directoryButton} + {previewFile && previewFile['id'] && + }
{tagsDisplay}
diff --git a/app/src/components/Map.js b/app/src/components/Map.js index 8bbc10de..f9ac7126 100644 --- a/app/src/components/Map.js +++ b/app/src/components/Map.js @@ -29,6 +29,7 @@ import {Media, StatusContext} from "../contexts/contexts"; import {MAP_VIEWER_URI} from "./Vars"; import {Modal} from "./Theme"; import MapViewer from "./MapViewer"; +import {AddToPlaylistButton} from "./AddToPlaylist"; import maplibregl from "maplibre-gl"; import {Protocol} from "pmtiles"; import layers from "protomaps-themes-base"; @@ -532,6 +533,12 @@ function MapPins() { {pin.created} + + + +
; + + const createModal = setModalOpen(false)} size='tiny'> + New Playlist + +
+ + + setName(value)}/> + +
+
+ + + + +
; + + return <> + {header} + {createModal} + + ; +} + + +// Icon (name + color) for a playlist item. File items use the FileGroup's mimetype/model icon +// and color (video, pdf, ebook, image, ...) so they match the rest of the UI; zim/url are fixed. +function itemIcon(item) { + if (item.item_kind === 'file' && item.file_group) { + const fg = item.file_group; + const lowerPath = (fg.primary_path || '').toLowerCase(); + return {name: mimetypeIconName(fg.mimetype, lowerPath), color: mimetypeColor(fg.mimetype, lowerPath)}; + } + if (item.item_kind === 'zim') return {name: 'book'}; + if (item.item_kind === 'url') { + // A map-location URL (e.g. /map?lat=&lon=) gets a map marker; other URLs get the link icon. + const u = (item.url || '').toLowerCase(); + if (u.startsWith('/map')) return {name: 'map marker alternate'}; + return {name: 'linkify'}; + } + return {name: 'file'}; +} + + +function itemLabel(item) { + if (item.item_kind === 'file' && item.file_group) { + return item.title || item.file_group.title || item.file_group.primary_path || 'File'; + } + if (item.item_kind === 'zim' && item.zim) { + return item.title || item.zim.entry; + } + if (item.item_kind === 'url') { + return item.title || item.url; + } + return item.title || 'Item'; +} + + +// Only http(s) and relative WROLPi paths are safe for an href; reject javascript:/data:/etc. so a +// shared playlist's url item cannot execute script when opened. Exported for tests. +export function safeHref(url) { + if (!url) return null; + try { + const u = new URL(url, window.location.origin); + if (u.protocol === 'http:' || u.protocol === 'https:') { + return url; // Keep the original (may be a relative WROLPi path). + } + } catch { + return null; + } + return null; +} + + +function itemLink(item) { + if (item.item_kind === 'url') { + return safeHref(item.url); + } + if (item.item_kind === 'zim' && item.zim) { + // Built from a numeric id + encoded entry — no user-supplied scheme. + return `/api/zim/${item.zim.id}/entry/${encodeURIComponent(item.zim.entry)}`; + } + return null; +} + + +// The in-app model page for a FileGroup, or null when there is none (image, audio, 3D, ...). +function fileModelUrl(fg) { + if (!fg) return null; + if (fg.model === 'video') return `/videos/${fg.id}`; + if (fg.model === 'archive') return `/archives/${fg.id}`; + if (fg.model === 'doc') return `/docs/${fg.id}`; + return null; +} + + +// Render a playlist item's title as the appropriate link: +// - file with a model page (video/archive/doc) -> in-app Link to that page +// - file without a model page (image, etc.) -> PreviewLink (opens the preview modal) +// - zim/url -> the entry/url in a new tab +function ItemTitle({item, label}) { + if (item.item_kind === 'file' && item.file_group) { + const url = fileModelUrl(item.file_group); + if (url) { + return {label}; + } + return {label}; + } + const link = itemLink(item); + return link + ? {label} + : {label}; +} + + +// The ordered items table, shared by the view and edit pages. When `editable`, the reorder/remove +// action buttons (and their column) are shown; otherwise the table is read-only. +function PlaylistItemsTable({items, editable, onMove, onRemove}) { + return + + + # + Item + Title + {editable && } + + + + {items.map((item, index) => { + const label = itemLabel(item); + const poster = item.item_kind === 'file' && item.file_group + ? findPosterPath(item.file_group) : null; + return + {String(index + 1).padStart(2, '0')} + + {poster + ? + : } + + + + + {editable && +
; +} + + +// View-only playlist page (opened from the "Name" column). No reorder/remove/add/delete controls; +// just an "Edit" button to switch to the edit page. +export function PlaylistViewPage() { + const {playlistId} = useParams(); + const {playlist} = usePlaylist(playlistId); + const {t} = React.useContext(ThemeContext); + + useTitle(playlist && playlist.name ? `${playlist.name} Playlist` : 'Playlist'); + + if (playlist === undefined) { + return Could not fetch playlist; + } + if (playlist === null) { + return ; + } + + const items = playlist.items || []; + + return <> + + +
+ {playlist.name} + + + +
+ {playlist.description &&

{playlist.description}

} + + {items.length === 0 + ? This playlist is empty + : } + ; +} + + +// Edit playlist page (opened from the edit icon), mirroring ChannelEditPage: Back/View buttons at +// the top, a form segment for name/description/tag, then a segment with the ordered items. +export function PlaylistEditPage() { + const {playlistId} = useParams(); + const {playlist, refetch} = usePlaylist(playlistId); + const navigate = useNavigate(); + const wrolMode = useWROLMode(); + const [url, setUrl] = useState(''); + const [urlTitle, setUrlTitle] = useState(''); + const [name, setName] = useState(''); + const [description, setDescription] = useState(''); + const [directory, setDirectory] = useState(''); + const [tagModalOpen, setTagModalOpen] = useState(false); + + useTitle(playlist && playlist.name ? `Edit ${playlist.name} Playlist` : 'Edit Playlist'); + + // Seed the form fields once the playlist loads (keyed on id so a refetch doesn't clobber edits). + const playlistDbId = playlist && playlist.id; + useEffect(() => { + if (playlistDbId) { + setName(playlist.name || ''); + setDescription(playlist.description || ''); + setDirectory(playlist.directory || ''); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [playlistDbId]); + + if (playlist === undefined) { + return Could not fetch playlist; + } + if (playlist === null) { + return ; + } + + const items = playlist.items || []; + // Editing is blocked while in WROL Mode (the API enforces this too); fall back to a read-only view. + const editable = !wrolMode; + + const handleSave = async () => { + try { + await updatePlaylist(playlistId, { + name: name.trim(), + description: description.trim(), + directory: directory.trim(), + }); + toast({ + type: 'success', title: 'Playlist Updated', + description: 'Playlist was successfully updated', time: 3000, + }); + await refetch(); + } catch (e) { + // Error toast already shown by the API client. + } + }; + + // Set or clear (tagName=null) the playlist's tag from the tag modal. `directory` is the + // modal's move-to suggestion (null when the move toggle is off). + const handleTagSave = async (tagName, directory) => { + try { + await setPlaylistTag(playlistId, tagName || '', directory); + await refetch(); + } catch (e) { + } + }; + + const handleAddUrl = async () => { + const trimmed = url.trim(); + if (!trimmed) return; + try { + await addPlaylistItem(playlistId, { + item_kind: 'url', url: trimmed, title: urlTitle.trim() || null, + }); + setUrl(''); + setUrlTitle(''); + await refetch(); + } catch (e) { + // Error toast already shown. + } + }; + + const handleRemove = async (itemId) => { + try { + await removePlaylistItem(playlistId, itemId); + await refetch(); + } catch (e) { + } + }; + + const move = async (index, delta) => { + const ids = items.map(i => i.id); + const target = index + delta; + if (target < 0 || target >= ids.length) return; + [ids[index], ids[target]] = [ids[target], ids[index]]; + try { + await reorderPlaylistItems(playlistId, ids); + await refetch(); + } catch (e) { + } + }; + + const handleDelete = async () => { + try { + await deletePlaylist(playlistId); + navigate('/playlists'); + } catch (e) { + } + }; + + const deleteButton = Delete; + + const tagButton = ; + + const actionButtons = <> + {deleteButton} + {tagButton} + ; + + // Minimal form object for CollectionEditForm (mirrors the useForm interface it consumes). + const form = {error: null, loading: false, disabled: !editable || !name.trim(), ready: true}; + + return <> + + + + + + + + + + + setName(value)} + /> + + + + + + setDirectory(value || '')} + /> + + + + + + + + setDescription(value)} + /> + + + + + + {/* Tag Modal */} + setTagModalOpen(false)} + currentTagName={playlist.tag_name} + originalDirectory={playlist.directory || ''} + getTagInfo={(tagName) => getCollectionTagInfo(playlistId, tagName)} + onSave={handleTagSave} + collectionName="Playlist" + /> + + {/* Items Segment */} + +
Items
+ + {items.length === 0 + ? + This playlist is empty + {editable && Add a link below.} + + : } + +
Add a link
+
+ + + setUrl(value)}/> + + + setUrlTitle(value)}/> + + + + + +
+
+ ; +} + + +export function PlaylistsRoute() { + return + + }/> + }/> + }/> + + ; +} diff --git a/app/src/components/Playlists.test.js b/app/src/components/Playlists.test.js new file mode 100644 index 00000000..b762b892 --- /dev/null +++ b/app/src/components/Playlists.test.js @@ -0,0 +1,43 @@ +import {safeHref} from './Playlists'; + +// safeHref is the only thing standing between a stored playlist url item and an executable href, +// so every dangerous-scheme vector must return null. +describe('safeHref', () => { + it('allows http(s) and relative WROLPi paths', () => { + expect(safeHref('/map?lat=40.76&lon=-111.89&z=12')).toBe('/map?lat=40.76&lon=-111.89&z=12'); + expect(safeHref('/api/zim/1/entry/home')).toBe('/api/zim/1/entry/home'); + expect(safeHref('relative/path')).toBe('relative/path'); + expect(safeHref('http://example.com/page')).toBe('http://example.com/page'); + expect(safeHref('https://example.com/page')).toBe('https://example.com/page'); + }); + + it('rejects empty values', () => { + expect(safeHref('')).toBeNull(); + expect(safeHref(null)).toBeNull(); + expect(safeHref(undefined)).toBeNull(); + }); + + it('rejects dangerous schemes', () => { + expect(safeHref('javascript:alert(1)')).toBeNull(); + expect(safeHref('data:text/html,')).toBeNull(); + expect(safeHref('vbscript:msgbox(1)')).toBeNull(); + expect(safeHref('blob:https://example.com/uuid')).toBeNull(); + expect(safeHref('file:///etc/passwd')).toBeNull(); + }); + + it('rejects case and whitespace smuggling tricks', () => { + expect(safeHref('JaVaScRiPt:alert(1)')).toBeNull(); + expect(safeHref(' javascript:alert(1)')).toBeNull(); + expect(safeHref('\tjavascript:alert(1)')).toBeNull(); + // The URL parser strips tab/newline anywhere, exactly as the browser does when resolving + // the href -- so this must be detected as a javascript: URL and rejected. + expect(safeHref('java\tscript:alert(1)')).toBeNull(); + expect(safeHref('java\nscript:alert(1)')).toBeNull(); + }); + + it('treats percent-encoded schemes as harmless relative paths', () => { + // Browsers do not percent-decode before scheme detection; this resolves as a relative + // path, which is safe to keep. + expect(safeHref('javascript%3Aalert(1)')).toBe('javascript%3Aalert(1)'); + }); +}); diff --git a/app/src/components/Theme.tsx b/app/src/components/Theme.tsx index 176143e4..11bc200b 100644 --- a/app/src/components/Theme.tsx +++ b/app/src/components/Theme.tsx @@ -15,6 +15,12 @@ import { FormInput as SFormInput, Header as SHeader, Icon as SIcon, + List as SList, + ListContent as SListContent, + ListDescription as SListDescription, + ListHeader as SListHeader, + ListIcon as SListIcon, + ListItem as SListItem, Loader as SLoader, Menu as SMenu, Modal as SModal, @@ -52,6 +58,12 @@ import { FormInputProps, HeaderProps, IconProps, + ListProps, + ListItemProps, + ListIconProps, + ListContentProps, + ListHeaderProps, + ListDescriptionProps, LoaderProps, MenuProps, ModalProps, @@ -398,6 +410,39 @@ export const Modal: ModalComponent = Object.assign(ModalBase, { Header: ModalHeader, }); +// ---------------------------------------------------------------------------- +// List Compound Component +// ---------------------------------------------------------------------------- +// Semantic's List supports `inverted`; the inverted class on the parent handles the item text, +// but List.Icon must also receive `inverted` so its color flips in dark mode (the parent's +// inverted class alone does not restyle the icon). + +interface ListComponent extends React.FC { + Item: React.FC; + Icon: React.FC; + Content: React.FC; + Header: React.FC; + Description: React.FC; +} + +const ListBase: React.FC = (props) => { + const {i} = useContext(ThemeContext); + return +}; + +const ListIcon: React.FC = (props) => { + const {i} = useContext(ThemeContext); + return +}; + +export const List: ListComponent = Object.assign(ListBase, { + Item: SListItem, + Icon: ListIcon, + Content: SListContent, + Header: SListHeader, + Description: SListDescription, +}); + // ---------------------------------------------------------------------------- // Confirm Component // ---------------------------------------------------------------------------- diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index 734d2909..644846e2 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -25,6 +25,7 @@ import {ThemeContext} from "../contexts/contexts"; import {Button, darkTheme, Header, Icon, Loader, Segment, Tab} from "./Theme"; import {VideoCard} from "./Videos"; import {TagsSelector} from "../Tags"; +import {AddToPlaylistButton} from "./AddToPlaylist"; import {Comment, CommentGroup, Input, Label, Transition} from "semantic-ui-react"; import {TaggedDeleteConfirmModal} from "./TaggedDeleteConfirmModal"; @@ -568,6 +569,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { obeyWROLMode={true} disabled={!videoFile.url} >Refresh +

diff --git a/app/src/components/Zim.js b/app/src/components/Zim.js index a6db633c..565d0608 100644 --- a/app/src/components/Zim.js +++ b/app/src/components/Zim.js @@ -45,6 +45,7 @@ import { } from "./Common"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import {TagsSelector} from "../Tags"; +import {AddToPlaylistButton} from "./AddToPlaylist"; import { deleteOutdatedZims, fetchZims, @@ -201,10 +202,13 @@ const ZimSearchEntry = ({zimId, onTag, onUntag, entry}) => { - + - + + ]*>/g, '')}} + content={null} title='Add to Playlist'/> diff --git a/app/src/components/admin/Configs.js b/app/src/components/admin/Configs.js index 1519ea80..86a43c1e 100644 --- a/app/src/components/admin/Configs.js +++ b/app/src/components/admin/Configs.js @@ -24,6 +24,7 @@ const TYPE_LABELS = { domain: 'Domains', download: 'Downloads', inventory: 'Inventories', + playlist: 'Playlists', }; const TYPE_COLUMNS = { @@ -34,6 +35,7 @@ const TYPE_COLUMNS = { domain: [{key: 'name', header: 'Name'}, {key: 'directory', header: 'Directory'}], download: [{key: 'url', header: 'URL'}, {key: 'downloader', header: 'Downloader'}], inventory: [{key: 'name', header: 'Name'}], + playlist: [{key: 'name', header: 'Name'}, {key: 'items', header: 'Items'}], }; function groupItemsByType(items) { diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index d8b4231e..c73b1548 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -473,6 +473,7 @@ export function SettingsPage() { throttle_on_startup: settings.throttle_on_startup, videos_destination: settings.videos_destination, zims_destination: settings.zims_destination, + playlists_destination: settings.playlists_destination, save_ffprobe_json: settings.save_ffprobe_json, }); }, [JSON.stringify(settings)]); @@ -862,6 +863,19 @@ export function SettingsPage() { + + + + + handleInputChange(e, 'playlists_destination', d.value)} + /> + + + /items') +@openapi.definition( + summary='Add an item (file, zim, or url) to a collection', + body=schema.AddItemRequest, + validate=True, +) +@openapi.response(HTTPStatus.CREATED, description="Item added") +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) +@wrol_mode_check +async def add_collection_item_endpoint(request: Request, collection_id: int, body: schema.AddItemRequest): + """Append (or insert at ``position``) a file/zim/url item into the collection.""" + session = request.ctx.session + try: + item, created = lib.add_collection_item( + session, collection_id, item_kind=body.item_kind, + file_group_id=body.file_group_id, zim_id=body.zim_id, zim_entry=body.zim_entry, + url=body.url, title=body.title, position=body.position) + except UnknownCollection as e: + return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND) + except ValidationError as e: + return json_response({'error': str(e)}, status=HTTPStatus.BAD_REQUEST) + + # 201 when a new item was created; 200 when the item already existed (idempotent add). + status = HTTPStatus.CREATED if created else HTTPStatus.OK + return json_response({'item': item.dict()}, status=status) + + +@collection_bp.delete('//items/') +@openapi.summary('Remove an item from a collection') +@openapi.response(HTTPStatus.NO_CONTENT, description="Item removed") +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +@wrol_mode_check +async def remove_collection_item_endpoint(request: Request, collection_id: int, item_id: int): + """Remove a single item by its id and close the ordering gap.""" + session = request.ctx.session + try: + removed = lib.remove_collection_item(session, collection_id, item_id) + except UnknownCollection as e: + return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND) + + if not removed: + return json_response({'error': f'Item {item_id} not found in collection {collection_id}'}, + status=HTTPStatus.NOT_FOUND) + return response.raw('', HTTPStatus.NO_CONTENT) + + +@collection_bp.put('//items/order') +@openapi.definition( + summary='Reorder a collection\'s items', + body=schema.ReorderItemsRequest, + validate=True, +) +@openapi.response(HTTPStatus.OK, description="Items reordered") +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) +@wrol_mode_check +async def reorder_collection_items_endpoint(request: Request, collection_id: int, + body: schema.ReorderItemsRequest): + """Set the item order from a full list of item ids (used by drag-and-drop).""" + session = request.ctx.session + try: + lib.reorder_collection_items(session, collection_id, body.item_ids) + except UnknownCollection as e: + return json_response({'error': str(e)}, status=HTTPStatus.NOT_FOUND) + except ValueError as e: + return json_response({'error': str(e)}, status=HTTPStatus.BAD_REQUEST) + + data = lib.get_collection_with_stats(session, collection_id) + return json_response({'collection': data}) + + @collection_bp.get('/') @openapi.summary('Get a single collection by ID') @openapi.response(HTTPStatus.OK, description="Collection details with statistics") @@ -102,7 +197,8 @@ async def put_collection_endpoint(request: Request, collection_id: int, body: sc collection_id=collection_id, directory=body.directory, tag_name=body.tag_name, - description=body.description + description=body.description, + name=body.name, ) # Return updated collection data diff --git a/wrolpi/collections/config.py b/wrolpi/collections/config.py index a570a70d..4ab1101e 100644 --- a/wrolpi/collections/config.py +++ b/wrolpi/collections/config.py @@ -1,126 +1,153 @@ import pathlib +import shutil from dataclasses import dataclass, field from typing import List -from wrolpi.common import ConfigFile, logger +from wrolpi.common import ConfigFile, get_media_directory, get_relative_to_media_directory, logger from wrolpi.db import get_db_session from wrolpi.events import Events -from wrolpi.switches import register_switch_handler, ActivateSwitchMethod +from wrolpi.switches import ActivateSwitchMethod, register_switch_handler logger = logger.getChild(__name__) -__all__ = ['CollectionsConfig', 'collections_config', 'save_collections_config'] +__all__ = ['PlaylistsConfig', 'playlists_config', 'save_playlists_config', 'get_playlists_config'] @dataclass -class CollectionsConfigValidator: - """Validator for collections config file.""" +class PlaylistsConfigValidator: + """Validator for the playlists config file.""" version: int = 0 - collections: List[dict] = field(default_factory=list) + playlists: List[dict] = field(default_factory=list) -class CollectionsConfig(ConfigFile): - """ - Config file for Collections. +class PlaylistsConfig(ConfigFile): + """Config file for playlists (Collections with kind='playlist'). - This config stores Collection metadata (name, description, directory, tag) - but NOT the individual items. Items are stored in the database. - - For directory-restricted collections, items are auto-populated from the directory. - For unrestricted collections, items are managed manually through the API. + Unlike other collection kinds, playlists are user-curated and ordered, with no other config + that owns them — so this config stores each playlist's metadata AND its ordered item list + (files/zim-articles/urls), making the config the source of truth that survives a DB rebuild. Format: - collections: - - name: "My Videos" - description: "Favorite videos" - directory: "videos/favorites" # optional, enables auto-population - tag_name: "favorites" # optional - - name: "Reading List" - description: "Books to read" - # No directory means manual item management + playlists: + - name: "Fire Making" + description: "..." + tag_name: "..." # optional + items: + - file: "videos/foo.mp4" + - zim: "zims/wikipedia.zim" + entry: "A/Fire" + title: "Making Fire" + - url: "/map?lat=1&lon=2&z=10" + title: "The spot" """ - file_name = 'collections.yaml' - validator = CollectionsConfigValidator - default_config = dict( - version=0, - collections=[], - ) - # Use wider width to accommodate longer paths + file_name = 'playlists.yaml' + validator = PlaylistsConfigValidator + default_config = dict(version=0, playlists=[]) width = 120 - def __getitem__(self, item): - return self._config[item] - - def __setitem__(self, key, value): - self._config[key] = value - @property - def collections(self) -> List[dict]: - """Get list of collection configs.""" - return self._config.get('collections', []) + def playlists(self) -> List[dict]: + return self._config.get('playlists', []) def import_config(self, file: pathlib.Path = None, send_events=False): - """Import collections from config file into database.""" - from .models import Collection + """Import playlists (and their ordered items) from the config into the database.""" + from modules.zim.models import Zim + from wrolpi.files.models import FileGroup + from wrolpi.tags import Tag - super().import_config(file, send_events) + from .models import Collection, CollectionItem + file = file or self.get_file() file_str = str(self.get_relative_file()) - collections_data = self._config.get('collections', []) - if not collections_data: - logger.info(f'No collections to import from {file_str}') + # A missing config is NOT an empty config (mirrors ChannelsConfig). Deleting here would + # cascade: the sync prunes the on-disk playlist dirs and the next dump writes an empty + # config -- a single lost file would permanently destroy every playlist. + if not file.is_file(): + logger.info(f'No playlists config file, skipping import: {file_str}') self.successful_import = True return - logger.info(f'Importing {len(collections_data)} collections from {file_str}') + super().import_config(file, send_events) + playlists_data = self._config.get('playlists', []) + + # An empty playlists list never deletes DB records (mirrors ChannelsConfig). Deleting all + # playlists is done through the API, which dumps the resulting (empty) config itself. + if not playlists_data: + logger.info(f'No playlists in config, preserving existing DB playlists: {file_str}') + self.successful_import = True + return try: with get_db_session(commit=True) as session: - # Track which collections were imported - imported_dirs = set() # absolute paths for directory-restricted collections - imported_pairs = set() # (name, kind) for unrestricted collections - - # Import each collection using from_config - for idx, collection_data in enumerate(collections_data): - try: - name = collection_data.get('name') - if not name: - logger.error(f'Collection at index {idx} has no name, skipping') - continue - - # Use Collection.from_config to create/update - # This will auto-populate items if directory exists - collection = Collection.from_config(collection_data, session) - - if collection.directory: - # Normalize to absolute string for comparison - imported_dirs.add(str(collection.directory)) - else: - imported_pairs.add((collection.name, collection.kind)) - - except Exception as e: - logger.error(f'Failed to import collection at index {idx}', exc_info=e) + imported_names = set() + for data in playlists_data: + name = data.get('name') + if not name: + logger.error(f'Playlist config entry has no name, skipping: {data}') continue - # Delete collections that are no longer in config - all_collections = session.query(Collection).all() - for collection in all_collections: - if collection.directory: - if str(collection.directory) not in imported_dirs: - logger.info( - f'Deleting collection {repr(collection.name)} at {collection.directory} (no longer in config)') - session.delete(collection) - else: - if (collection.name, collection.kind) not in imported_pairs: - logger.info( - f"Deleting unrestricted collection {repr(collection.name)} kind={collection.kind} (no longer in config)") - session.delete(collection) + collection = session.query(Collection).filter_by( + name=name, kind='playlist').one_or_none() + if not collection: + collection = Collection(name=name, kind='playlist') + session.add(collection) + collection.description = data.get('description') - total_imported = len(imported_dirs) + len(imported_pairs) - logger.info(f'Successfully imported {total_imported} collections from {file_str}') + # Optional custom directory (media-relative); None when not customized. + directory = data.get('directory') + collection.directory = (get_media_directory() / directory) if directory else None + + tag_name = data.get('tag_name') + collection.tag = Tag.get_by_name(session, tag_name) if tag_name else None + session.flush([collection]) + + # Rebuild items to match the config's order exactly. + for item in list(collection.items): + session.delete(item) + session.flush() + + position = 1 + for item_data in (data.get('items') or []): + item = None + if 'file' in item_data: + path = get_media_directory() / item_data['file'] + fg = session.query(FileGroup).filter_by(primary_path=str(path)).one_or_none() + if fg: + item = CollectionItem(collection_id=collection.id, item_kind='file', + file_group_id=fg.id) + else: + logger.warning(f'Playlist {name!r}: file not indexed, skipping ' + f'{item_data["file"]!r}') + elif 'zim' in item_data: + zpath = get_media_directory() / item_data['zim'] + zim = session.query(Zim).filter_by(path=str(zpath)).one_or_none() + if zim: + item = CollectionItem(collection_id=collection.id, item_kind='zim', + zim_id=zim.id, zim_entry=item_data.get('entry')) + else: + logger.warning(f'Playlist {name!r}: zim not found, skipping ' + f'{item_data["zim"]!r}') + elif 'url' in item_data: + item = CollectionItem(collection_id=collection.id, item_kind='url', + url=item_data['url']) + if item is not None: + item.title = item_data.get('title') + item.position = position + position += 1 + session.add(item) + + session.flush() + imported_names.add(name) + + # Delete playlists that are no longer present in the config. + for collection in session.query(Collection).filter_by(kind='playlist').all(): + if collection.name not in imported_names: + logger.info(f'Deleting playlist {collection.name!r} (no longer in config)') + session.delete(collection) + + logger.info(f'Imported {len(imported_names)} playlists from {file_str}') self.successful_import = True - except Exception as e: self.successful_import = False message = f'Failed to import {file_str} config!' @@ -129,35 +156,134 @@ class CollectionsConfig(ConfigFile): Events.send_config_import_failed(message) raise + def preview_backup_import(self, backup_date: str, mode: str) -> dict: + """Preview which playlists a backup import would add/remove (keyed by playlist name).""" + backup_file = self._get_backup_file(backup_date) + backup_data = self.read_config_file(backup_file) + current_data = self.read_config_file() if self.get_file().is_file() else dict(playlists=[], version=0) + + backup_playlists = backup_data.get('playlists', []) + current_playlists = current_data.get('playlists', []) + + current_names = {p.get('name') for p in current_playlists if p.get('name')} + backup_names = {p.get('name') for p in backup_playlists if p.get('name')} + + add = [] + remove = [] + unchanged = 0 + + for playlist in backup_playlists: + name = playlist.get('name') + if name and name not in current_names: + add.append(dict(type='playlist', name=name, items=len(playlist.get('items') or []))) + elif name: + unchanged += 1 + if mode == 'overwrite': + for name in sorted(current_names - backup_names): + remove.append(dict(type='playlist', name=name)) + + return dict(mode=mode, add=add, remove=remove, unchanged=unchanged) + + def import_backup(self, backup_date: str, mode: str, send_events: bool = False): + """Restore playlists from a dated backup of playlists.yaml. + + 'overwrite' replaces the config with the backup; 'merge' adds backup playlists whose names + are not already in the current config. Either way the result is imported into the DB. + """ + self._preserve_current_config() + backup_file = self._get_backup_file(backup_date) + config_file = self.get_file() + + if mode == 'overwrite': + config_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(backup_file, config_file) + elif mode == 'merge': + backup_data = self.read_config_file(backup_file) + current_data = self.read_config_file() if config_file.is_file() else dict(playlists=[], version=0) + + current_names = {p.get('name') for p in current_data.get('playlists', []) if p.get('name')} + merged_playlists = list(current_data.get('playlists', [])) + for playlist in backup_data.get('playlists', []): + if playlist.get('name') and playlist['name'] not in current_names: + merged_playlists.append(playlist) + + merged_data = dict( + playlists=merged_playlists, + version=current_data.get('version', 0) + 1, + ) + config_file.parent.mkdir(parents=True, exist_ok=True) + self.write_config_data(merged_data, config_file) + + self.import_config(send_events=send_events) + # Materialize the restored playlists on disk. + from .sync import sync_playlists_directory + sync_playlists_directory.activate_switch() + def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False): - """Dump all collections from database to config file.""" + """Dump all playlists (with their ordered items) to the config file.""" from .models import Collection - logger.info('Dumping collections to config') + # playlists.yaml is derived from the database (the source of truth here), so the dump should + # always win. Sync our in-memory version up to the on-disk version first, otherwise a Sanic + # worker (or freshly-restarted process) that hasn't re-imported would refuse to overwrite a + # newer on-disk config. + target = file or self.get_file() + try: + if target.exists(): + disk_version = (self.read_config_file(target) or {}).get('version') or 0 + if disk_version > (self._config.get('version') or 0): + self._config['version'] = disk_version + except Exception as e: + logger.warning(f'Could not read existing playlists config version: {e}') with get_db_session() as session: - # Order by name for consistency - collections = session.query(Collection).order_by(Collection.name).all() + playlists = session.query(Collection).filter_by(kind='playlist') \ + .order_by(Collection.name).all() + data = [] + for collection in playlists: + entry = {'name': collection.name} + if collection.description: + entry['description'] = collection.description + if collection.tag_name: + entry['tag_name'] = collection.tag_name + if collection.directory: + # Custom playlist directory; stored relative to the media directory. + entry['directory'] = str(get_relative_to_media_directory(collection.directory)) + entry['items'] = [c for c in (i.to_config() for i in collection.items) if c] + data.append(entry) + self._config['playlists'] = data - # Use to_config to export each collection - collections_data = [collection.to_config() for collection in collections] - - self._config['collections'] = collections_data - - logger.info(f'Dumping {len(collections_data)} collections to config') + logger.info(f'Dumping {len(data)} playlists to config') self.save(file, send_events, overwrite) # Global instance -collections_config = CollectionsConfig() +playlists_config = PlaylistsConfig() -# Switch handler for saving collections config -@register_switch_handler('save_collections_config') -def save_collections_config(): - """Save the collections config when the switch is activated.""" - collections_config.background_dump.activate_switch() +@register_switch_handler('save_playlists_config') +def save_playlists_config(): + """Save the playlists config when the switch is activated.""" + playlists_config.background_dump.activate_switch() -# Explicit type for activate_switch helper -save_collections_config: ActivateSwitchMethod +def get_playlists_config() -> PlaylistsConfig: + return playlists_config + + +@register_switch_handler('import_playlists_config') +def import_playlists_config(): + """Re-import playlists.yaml into the database. + + Activated when a global refresh completes: after a DB rebuild, playlists.yaml is imported at + startup before any files are indexed, so its file/zim items are skipped. The config still + holds them; once the refresh has re-indexed the files, this re-import restores those items. + """ + playlists_config.import_config() + # Restored items need their hardlinks/stubs re-created. + from .sync import sync_playlists_directory + sync_playlists_directory.activate_switch() + + +save_playlists_config: ActivateSwitchMethod +import_playlists_config: ActivateSwitchMethod diff --git a/wrolpi/collections/errors.py b/wrolpi/collections/errors.py index d5a60452..997e9e23 100644 --- a/wrolpi/collections/errors.py +++ b/wrolpi/collections/errors.py @@ -1,14 +1,8 @@ -from wrolpi.errors import APIError +from wrolpi.errors import APIError, UnknownCollection __all__ = ['UnknownCollection', 'ReorganizationConflict'] -class UnknownCollection(APIError): - """Cannot find Collection""" - code = 'UNKNOWN_COLLECTION' - summary = 'Cannot find Collection' - - class ReorganizationConflict(APIError): """Reorganization would cause filename conflicts""" code = 'REORGANIZATION_CONFLICT' diff --git a/wrolpi/collections/lib.py b/wrolpi/collections/lib.py index 061da730..596dde1f 100644 --- a/wrolpi/collections/lib.py +++ b/wrolpi/collections/lib.py @@ -17,8 +17,8 @@ from wrolpi.errors import FileWorkerConflict from wrolpi.errors import ValidationError from wrolpi.events import Events from wrolpi.tags import Tag -from .errors import UnknownCollection from .models import Collection, validate_collection_directory +from .types import collection_type_registry logger = logger.getChild(__name__) @@ -26,6 +26,10 @@ __all__ = [ 'get_collections', 'get_collection_with_stats', 'get_domain_statistics', + 'create_collection', + 'add_collection_item', + 'remove_collection_item', + 'reorder_collection_items', 'update_collection', 'refresh_collection', 'tag_collection', @@ -35,6 +39,23 @@ __all__ = [ ] +def _activate_config_save(kind: str): + """Activate the config-save switch for the config that owns this kind of Collection. + + Author/subject collections are re-derived on refresh and have no config.""" + if kind == 'domain': + # Local import to avoid circular import: collections -> archive -> collections + from modules.archive.lib import save_domains_config + save_domains_config.activate_switch() + elif kind == 'channel': + # Local import to avoid circular import: collections -> videos -> collections + from modules.videos.lib import save_channels_config + save_channels_config.activate_switch() + elif kind == 'playlist': + from .config import save_playlists_config + save_playlists_config.activate_switch() + + def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]: """ Get all collections, optionally filtered by kind. @@ -169,10 +190,7 @@ def get_collection_with_stats(session: Session, collection_id: int) -> dict: Raises: UnknownCollection: If collection not found """ - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() - - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + collection = Collection.find_by_id(session, collection_id) # Get base collection data data = collection.__json__() @@ -216,13 +234,17 @@ def get_collection_with_stats(session: Session, collection_id: int) -> dict: data['total_size'] = 0 else: - # Generic collections (author, subject, etc.): count actual CollectionItem rows. + # Generic collections (author, subject, playlist, etc.): count actual CollectionItem rows. from .models import CollectionItem item_count = session.query(func.count(CollectionItem.id)).filter( CollectionItem.collection_id == collection_id ).scalar() data['item_count'] = item_count or 0 + # Playlists are manually curated and ordered, so return the full ordered item list. + if collection.kind == 'playlist': + data['items'] = [item.dict() for item in collection.items] + return data @@ -232,6 +254,7 @@ def update_collection( directory: Optional[str] = None, tag_name: Optional[str] = None, description: Optional[str] = None, + name: Optional[str] = None, ) -> Collection: """ Update a collection's properties. @@ -242,6 +265,7 @@ def update_collection( directory: New directory (relative or absolute path), or None to clear tag_name: New tag name, empty string to clear, or None to leave unchanged description: New description, or None to leave unchanged + name: New name (rename), or None to leave unchanged Returns: Updated Collection object @@ -250,12 +274,27 @@ def update_collection( UnknownCollection: If collection not found ValidationError: If validation fails """ - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() + collection = Collection.find_by_id(session, collection_id) - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + # Rename if provided. + if name is not None: + name = name.strip() + if not name: + raise ValidationError('Collection name cannot be empty') + if name != collection.name: + if not collection_type_registry.validate(collection.kind, name): + raise ValidationError(f'Invalid {collection.kind} name {name!r}') + duplicate = session.query(Collection).filter( + Collection.name == name, + Collection.kind == collection.kind, + Collection.id != collection.id, + ).first() + if duplicate: + raise ValidationError(f'A {collection.kind} named {name!r} already exists') + collection.name = name # Update directory if provided + old_directory = collection.directory if directory is not None: if directory: # Convert relative path to absolute with validation @@ -271,33 +310,157 @@ def update_collection( # Update tag if provided if tag_name is not None: if tag_name: - # Set or create tag - if not collection.directory: + # Set or create tag. Playlists manage their own on-disk layout (the sync namespaces them + # under the tag), so they may be tagged without a `directory`; other kinds require one. + if collection.kind != 'playlist' and not collection.directory: raise ValidationError( f"Collection '{collection.name}' has no directory. " f"Set a directory before tagging." ) - tag = session.query(Tag).filter_by(name=tag_name).one_or_none() - if not tag: - tag = Tag(name=tag_name) - session.add(tag) - session.flush() + tag = Tag.get_or_create_tag(session, name=tag_name) + # Assign the relationship too so `collection.tag_name` is fresh below. + collection.tag = tag collection.tag_id = tag.id elif tag_name == '': # Clear tag (empty string explicitly clears) + collection.tag = None collection.tag_id = None session.flush() - # Trigger domain config save if this is a domain collection - if collection.kind == 'domain': - # Local import to avoid circular import: collections -> archive -> collections - from modules.archive.lib import save_domains_config - save_domains_config.activate_switch() + if collection.kind == 'playlist': + from .sync import (cleanup_playlist_directory, get_playlists_directory, + _default_playlist_subdir, sync_playlists_directory) + # A directory equal to the managed tag location is not "custom" -- store None so the sync + # keeps auto-managing it (future tag changes keep moving the playlist automatically). + if collection.directory and \ + pathlib.Path(collection.directory) == _default_playlist_subdir(get_playlists_directory(), collection): + collection.directory = None + session.flush() + # Commit before the cleanup and switch activations so the background sync reads the NEW + # state -- otherwise it can run against the pre-commit row, re-create the old directory, + # and undo the cleanup (same pattern as tag_collection). + session.commit() + # A custom directory lives outside the playlists root, so the sync's orphan pruning never + # sees it -- clean up the abandoned directory here when the playlist moves away from it. + if old_directory and str(old_directory) != str(collection.directory or ''): + cleanup_playlist_directory(old_directory) + # Re-sync the on-disk playlist (a tag/directory change moves it). + sync_playlists_directory.activate_switch() + + _activate_config_save(collection.kind) return collection +def create_collection(session: Session, name: str, description: Optional[str] = None, + kind: str = 'playlist') -> Collection: + """Create a new Collection (defaults to a 'playlist'). + + Raises: + ValidationError: if the name is empty/invalid for the kind, or a duplicate exists. + """ + name = (name or '').strip() + if not name: + raise ValidationError('Collection name is required') + if not collection_type_registry.is_registered(kind): + raise ValidationError(f'Unknown collection kind: {kind!r}') + if not collection_type_registry.validate(kind, name): + description_msg = collection_type_registry.get_description(kind) or 'Invalid name format' + raise ValidationError(f'Invalid {kind} name {name!r}. {description_msg}') + + existing = session.query(Collection).filter_by(name=name, kind=kind).first() + if existing: + raise ValidationError(f'A {kind} named {name!r} already exists') + + collection = Collection(name=name, description=description, kind=kind) + session.add(collection) + session.flush([collection]) + + _activate_config_save(kind) + return collection + + +def add_collection_item(session: Session, collection_id: int, item_kind: str, + file_group_id: Optional[int] = None, zim_id: Optional[int] = None, + zim_entry: Optional[str] = None, url: Optional[str] = None, + title: Optional[str] = None, position: Optional[int] = None): + """Add a file/zim/url item to a collection. + + Adding an item that already exists is idempotent: the existing CollectionItem is returned. + + Returns: + (CollectionItem, created): ``created`` is False when the item already existed. + + Raises: + UnknownCollection: if the collection does not exist. + ValidationError: if the item is invalid for its type. + """ + from .models import CollectionItem + collection = Collection.find_by_id(session, collection_id) + + if item_kind == 'file': + from wrolpi.files.models import FileGroup + if not file_group_id: + raise ValidationError('file_group_id is required for a file item') + file_group = session.query(FileGroup).filter_by(id=file_group_id).one_or_none() + if not file_group: + raise ValidationError(f'No FileGroup with id {file_group_id}') + created = session.query(CollectionItem).filter_by( + collection_id=collection_id, file_group_id=file_group_id).first() is None + try: + item = collection.add_file_group(file_group, position=position, session=session) + except ValueError as e: + raise ValidationError(str(e)) + if title: + item.title = title + elif item_kind == 'zim': + created = session.query(CollectionItem).filter_by( + collection_id=collection_id, zim_id=zim_id, zim_entry=zim_entry).first() is None + try: + item = collection.add_zim_entry(session, zim_id, zim_entry, title=title, position=position) + except ValueError as e: + raise ValidationError(str(e)) + elif item_kind == 'url': + created = session.query(CollectionItem).filter_by( + collection_id=collection_id, url=url).first() is None + try: + item = collection.add_url(session, url, title=title, position=position) + except ValueError as e: + raise ValidationError(str(e)) + else: + raise ValidationError(f'Unknown item_kind: {item_kind!r}') + + session.flush() + from .config import save_playlists_config + save_playlists_config.activate_switch() + from .sync import sync_playlists_directory + sync_playlists_directory.activate_switch() + return item, created + + +def remove_collection_item(session: Session, collection_id: int, item_id: int) -> bool: + """Remove an item from a collection by CollectionItem id. Returns True if removed.""" + collection = Collection.find_by_id(session, collection_id) + removed = collection.remove_item(session, item_id) + if removed: + from .config import save_playlists_config + save_playlists_config.activate_switch() + from .sync import sync_playlists_directory + sync_playlists_directory.activate_switch() + return removed + + +def reorder_collection_items(session: Session, collection_id: int, item_ids: List[int]): + """Reorder a collection's items from a full list of CollectionItem ids.""" + collection = Collection.find_by_id(session, collection_id) + collection.reorder_items(session, item_ids) + from .config import save_playlists_config + save_playlists_config.activate_switch() + from .sync import sync_playlists_directory + sync_playlists_directory.activate_switch() + + def _get_external_collection_file_paths(session, collection_id: int, directory: pathlib.Path) -> list[pathlib.Path]: """Return primary_path for FileGroups associated with a collection but outside its directory. @@ -351,10 +514,7 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None: from wrolpi.files.worker import file_worker with get_db_session() as session: - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() - - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + collection = Collection.find_by_id(session, collection_id) if not collection.directory: raise ValidationError( @@ -416,85 +576,17 @@ async def tag_collection( ValidationError: If tagging requirements not met FileWorkerConflict: If a file operation is in progress and directory change is requested """ - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() - - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + collection = Collection.find_by_id(session, collection_id) # Track old directory before any changes for potential file moving old_directory = collection.directory - # If tag_name is None, remove the tag from the collection - if tag_name is None: - collection.tag_id = None - - # Update directory if provided (user may want to move files when removing tag) - if directory: - target_directory = validate_collection_directory(directory) - else: - target_directory = collection.directory - - # Check if we need to move files - need_to_move = ( - target_directory is not None and - old_directory is not None and - target_directory != old_directory - ) - - if __debug__ and logger.isEnabledFor(TRACE_LEVEL): - logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, ' - f'{old_directory} -> {target_directory}') - - if need_to_move and flags.file_worker_busy.is_set(): - raise FileWorkerConflict('Refusing to move collection while FileWorker is busy') - - session.flush() - - # Trigger config save based on collection kind - if collection.kind == 'domain': - # Local import to avoid circular import: collections -> archive -> collections - from modules.archive.lib import save_domains_config - save_domains_config.activate_switch() - elif collection.kind == 'channel': - # Local import to avoid circular import: collections -> videos -> collections - from modules.videos.lib import save_channels_config - save_channels_config.activate_switch() - - # Move files if directory changed - run in background so closing tab won't cancel it - if need_to_move: - target_directory.mkdir(parents=True, exist_ok=True) - # Commit tag changes before starting background move - session.commit() - background_task(_background_move_collection(collection.id, target_directory)) - - # Return info about the un-tagging operation - relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None - return { - 'collection_id': collection.id, - 'collection_name': collection.name, - 'tag_name': None, - 'directory': str(relative_dir) if relative_dir else None, - 'will_move_files': need_to_move, - } - - # Get or create the tag - tag = session.query(Tag).filter_by(name=tag_name).one_or_none() - if not tag: - tag = Tag(name=tag_name) - session.add(tag) - session.flush() - - # Determine target directory + # Determine target directory. Collections can be tagged without a directory for UI + # search/filtering, so a missing directory is left missing. if directory: - # User specified a directory - validate it target_directory = validate_collection_directory(directory) - elif collection.directory: - # Use existing directory - target_directory = collection.directory else: - # No directory provided and collection has none - keep it that way - # Collections can be tagged without a directory for UI search/filtering - target_directory = None + target_directory = collection.directory # Check if we need to move files need_to_move = ( @@ -510,24 +602,19 @@ async def tag_collection( if need_to_move and flags.file_worker_busy.is_set(): raise FileWorkerConflict('Refusing to move collection while FileWorker is busy') - # Apply the tag - collection.tag_id = tag.id - # Only update directory if we have a target (don't auto-generate for directory-less collections) - # Note: directory update is handled by move_collection if need_to_move, otherwise set directly - if target_directory is not None and not need_to_move: - collection.directory = target_directory + # Apply (or remove) the tag + if tag_name is None: + collection.tag_id = None + else: + collection.tag_id = Tag.get_or_create_tag(session, name=tag_name).id + # Only update directory if we have a target (don't auto-generate for directory-less + # collections). Directory update is handled by move_collection if need_to_move. + if target_directory is not None and not need_to_move: + collection.directory = target_directory session.flush() - # Trigger config save based on collection kind - if collection.kind == 'domain': - # Local import to avoid circular import: collections -> archive -> collections - from modules.archive.lib import save_domains_config - save_domains_config.activate_switch() - elif collection.kind == 'channel': - # Local import to avoid circular import: collections -> videos -> collections - from modules.videos.lib import save_channels_config - save_channels_config.activate_switch() + _activate_config_save(collection.kind) # Move files if directory changed - run in background so closing tab won't cancel it if need_to_move: @@ -536,7 +623,6 @@ async def tag_collection( session.commit() background_task(_background_move_collection(collection.id, target_directory)) - # Return info about the tagging operation relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None return { 'collection_id': collection.id, @@ -569,10 +655,22 @@ def get_tag_info( Raises: UnknownCollection: If collection not found """ - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() + collection = Collection.find_by_id(session, collection_id) - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + # Playlists: suggest the managed tag location (//), even when the + # playlist has no explicit directory (most don't -- the sync manages their location). + if collection.kind == 'playlist': + from wrolpi.common import escape_file_name + from .sync import get_playlists_directory + base = get_playlists_directory() + if tag_name: + base = base / escape_file_name(tag_name) + suggested = base / escape_file_name(collection.name) + return { + 'suggested_directory': str(get_relative_to_media_directory(suggested)), + 'conflict': False, + 'conflict_message': None, + } # For collections without a directory, return None - no directory suggestions needed if collection.directory is None: @@ -643,10 +741,7 @@ def delete_collection( Raises: UnknownCollection: If collection not found """ - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() - - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + collection = Collection.find_by_id(session, collection_id) collection_dict = { 'id': collection.id, @@ -656,22 +751,17 @@ def delete_collection( # Orphan child Archives if this is a domain collection if collection.kind == 'domain': - # Local imports to avoid circular import: collections -> archive -> collections + # Local import to avoid circular import: collections -> archive -> collections from modules.archive.models import Archive - from modules.archive.lib import save_domains_config archives = session.query(Archive).filter_by(collection_id=collection_id).all() for archive in archives: archive.collection_id = None session.flush() - # Trigger domain config save - save_domains_config.activate_switch() - # Orphan child Videos and delete Channel if this is a channel collection elif collection.kind == 'channel': # Local imports to avoid circular import: collections -> videos -> collections from modules.videos.models import Channel, Video - from modules.videos.lib import save_channels_config # Find the Channel that references this Collection channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none() @@ -686,13 +776,27 @@ def delete_collection( session.delete(channel) session.flush() - # Trigger channel config save - save_channels_config.activate_switch() + # A playlist's custom directory lives outside the playlists root; remember it so its managed + # files can be cleaned up after the delete (the sync's orphan pruning only covers the root). + playlist_custom_directory = collection.directory if collection.kind == 'playlist' else None # Delete the collection session.delete(collection) session.flush() + # Clean up the playlist's on-disk directory and re-sync. + if collection_dict['kind'] == 'playlist': + from .sync import cleanup_playlist_directory, sync_playlists_directory + # Commit before the cleanup and switch activations so the background sync reads the NEW + # state (the deleted row); a sync of pre-commit state would re-create the directory. + session.commit() + if playlist_custom_directory: + cleanup_playlist_directory(playlist_custom_directory) + sync_playlists_directory.activate_switch() + + # Persist the deletion to the owning config. + _activate_config_save(collection_dict['kind']) + return collection_dict diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index 5549cf8e..848fbf74 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -2,7 +2,7 @@ import pathlib from typing import Optional, List from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func, BigInteger, \ - or_ + or_, CheckConstraint from sqlalchemy.orm import relationship, Session from sqlalchemy.orm.collections import InstrumentedList @@ -761,6 +761,105 @@ class Collection(ModelHelper, Base): ).update({'position': CollectionItem.position + shift_by}) session.flush() + def _next_position(self, session: Session) -> int: + """Return the position to use when appending a new item to the end.""" + max_position = session.query(func.max(CollectionItem.position)).filter_by( + collection_id=self.id + ).scalar() + return (max_position or 0) + 1 + + def _place_item(self, session: Session, item: 'CollectionItem', position: Optional[int]): + """Assign ``position`` to ``item`` (appending or inserting with a shift) and persist it.""" + if position is None: + item.position = self._next_position(session) + else: + self._shift_positions(session, position, shift_by=1) + item.position = position + session.add(item) + session.flush([item]) + + def add_url(self, session: Session, url: str, title: Optional[str] = None, + position: Optional[int] = None) -> 'CollectionItem': + """Add a `url` item (a link the WROLPi browser can open). Idempotent per URL.""" + if not url: + raise ValueError('A url item requires a url') + + existing = session.query(CollectionItem).filter_by(collection_id=self.id, url=url).first() + if existing: + logger.warning(f'URL {url!r} already in Collection {self.id}') + return existing + + item = CollectionItem(collection_id=self.id, item_kind='url', url=url, title=title) + self._place_item(session, item, position) + return item + + def add_zim_entry(self, session: Session, zim_id: int, zim_entry: str, title: Optional[str] = None, + position: Optional[int] = None) -> 'CollectionItem': + """Add a `zim` item (a single article within a Zim file). Idempotent per (zim, entry).""" + # Local import to avoid a circular import (zim -> tags -> collections). + from modules.zim.models import Zim + + zim = session.query(Zim).filter_by(id=zim_id).one_or_none() + if not zim: + raise ValueError(f'No Zim with id {zim_id}') + # Validate the entry exists within the Zim. + try: + zim.find_entry(zim_entry) + except Exception as e: + raise ValueError(f'Zim entry not found: {zim_entry!r}') from e + + existing = session.query(CollectionItem).filter_by( + collection_id=self.id, zim_id=zim_id, zim_entry=zim_entry).first() + if existing: + logger.warning(f'Zim entry {zim_entry!r} already in Collection {self.id}') + return existing + + item = CollectionItem(collection_id=self.id, item_kind='zim', zim_id=zim_id, + zim_entry=zim_entry, title=title) + self._place_item(session, item, position) + return item + + def remove_item(self, session: Session, item_id: int) -> bool: + """Remove an item by its CollectionItem id (any item_kind). Returns True if removed.""" + item = session.query(CollectionItem).filter_by(collection_id=self.id, id=item_id).first() + if not item: + return False + position = item.position + session.delete(item) + session.flush() + # Close the gap left behind. + self._shift_positions(session, position + 1, shift_by=-1) + return True + + def reorder_items(self, session: Session, ordered_item_ids: List[int]): + """Set item order from a full list of CollectionItem ids (for drag-and-drop reordering). + + Any items not present in ``ordered_item_ids`` are appended after, keeping their prior order. + + @raise ValueError: if an id does not belong to this collection. + """ + items = session.query(CollectionItem).filter_by(collection_id=self.id).all() + by_id = {i.id: i for i in items} + for item_id in ordered_item_ids: + if item_id not in by_id: + raise ValueError(f'Item {item_id} is not in Collection {self.id}') + + position = 1 + seen = set() + for item_id in ordered_item_ids: + if item_id in seen: + # Ignore duplicate ids so positions stay contiguous (no gaps). + continue + by_id[item_id].position = position + seen.add(item_id) + position += 1 + # Preserve any unlisted items after the explicitly-ordered ones. + for item in sorted(items, key=lambda i: i.position): + if item.id not in seen: + item.position = position + position += 1 + session.flush() + def get_items(self, session: Session, limit: Optional[int] = None, offset: int = 0) -> List['CollectionItem']: """Get items in this collection, ordered by position.""" @@ -1041,14 +1140,35 @@ class Collection(ModelHelper, Base): class CollectionItem(ModelHelper, Base): """ - Junction table between Collection and FileGroup. - Maintains ordering and metadata about the relationship. + An ordered item within a Collection. + + An item is one of three kinds (``item_kind``): + - ``file``: references a FileGroup (video, archive, ebook, image, doc, ...). + - ``zim``: references a single Zim article (``zim_id`` + ``zim_entry`` path), like TagZimEntry. + - ``url``: an arbitrary URL the WROLPi browser can open (e.g. a map location ``/map?lat=&lon=``). + + Exactly one kind's columns are populated; enforced by a CHECK constraint. """ __tablename__ = 'collection_item' + ITEM_KINDS = ('file', 'zim', 'url') + id = Column(Integer, primary_key=True) collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'), nullable=False) - file_group_id = Column(Integer, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False) + + # Which kind of item this is. + item_kind = Column(String, nullable=False, server_default='file', default='file') + + # `file` items: a FileGroup. + file_group_id = Column(Integer, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=True) + # `zim` items: a single article within a Zim file. + zim_id = Column(Integer, ForeignKey('zim.id', ondelete='CASCADE'), nullable=True) + zim_entry = Column(Text, nullable=True) + # `url` items: any URL. + url = Column(Text, nullable=True) + + # Display title (used for `url`/`zim` items, and as the on-disk stem; optional for `file`). + title = Column(Text, nullable=True) # Position in the collection (for ordering) position = Column(Integer, nullable=False, default=0) @@ -1059,6 +1179,7 @@ class CollectionItem(ModelHelper, Base): # Relationships collection = relationship('Collection', back_populates='items') file_group: FileGroup = relationship('FileGroup') + zim = relationship('Zim') # Indexes for performance __table_args__ = ( @@ -1066,15 +1187,56 @@ class CollectionItem(ModelHelper, Base): Index('idx_collection_item_collection_position', 'collection_id', 'position'), Index('idx_collection_item_file_group_id', 'file_group_id'), Index('idx_collection_item_position', 'position'), + # A FileGroup/Zim-article/URL appears at most once per collection. NULLs are distinct in + # Postgres, so each unique constraint only applies to items of that kind. UniqueConstraint('collection_id', 'file_group_id', name='uq_collection_file_group'), + UniqueConstraint('collection_id', 'zim_id', 'zim_entry', name='uq_collection_zim_entry'), + UniqueConstraint('collection_id', 'url', name='uq_collection_url'), + # Exactly one kind's columns are populated. + CheckConstraint( + "(item_kind = 'file' AND file_group_id IS NOT NULL AND zim_id IS NULL AND url IS NULL)" + " OR (item_kind = 'zim' AND zim_id IS NOT NULL AND zim_entry IS NOT NULL" + " AND file_group_id IS NULL AND url IS NULL)" + " OR (item_kind = 'url' AND url IS NOT NULL AND file_group_id IS NULL AND zim_id IS NULL)", + name='ck_collection_item_kind', + ), ) def __repr__(self): - return f'' + return (f'') def dict(self) -> dict: """Return dictionary representation.""" d = super(CollectionItem, self).dict() - if self.file_group: + if self.item_kind == 'file' and self.file_group: d['file_group'] = self.file_group.__json__() + elif self.item_kind == 'zim' and self.zim: + d['zim'] = {'id': self.zim_id, 'entry': self.zim_entry, 'path': str(self.zim.path)} return d + + def to_config(self) -> Optional[dict]: + """Serialize this item for playlists.yaml. + + Files/Zims are referenced by their media-relative path so they survive a DB rebuild. + Returns None if the item cannot currently be represented (e.g. missing relationship). + """ + if self.item_kind == 'file' and self.file_group: + try: + ref = str(get_relative_to_media_directory(self.file_group.primary_path)) + except ValueError: + ref = str(self.file_group.primary_path) + data = {'file': ref} + elif self.item_kind == 'zim' and self.zim: + try: + ref = str(get_relative_to_media_directory(self.zim.path)) + except ValueError: + ref = str(self.zim.path) + data = {'zim': ref, 'entry': self.zim_entry} + elif self.item_kind == 'url': + data = {'url': self.url} + else: + return None + if self.title: + data['title'] = self.title + return data diff --git a/wrolpi/collections/reorganize.py b/wrolpi/collections/reorganize.py index 5b48fc90..4f5700bb 100644 --- a/wrolpi/collections/reorganize.py +++ b/wrolpi/collections/reorganize.py @@ -17,7 +17,6 @@ from sqlalchemy.orm import Session, joinedload from wrolpi.common import logger, get_media_directory from wrolpi.db import get_db_session -from .errors import UnknownCollection from .models import Collection logger = logger.getChild(__name__) @@ -236,9 +235,7 @@ def get_reorganization_preview( with get_db_session() as session: return get_reorganization_preview(collection_id, session, sample_size) - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + collection = Collection.find_by_id(session, collection_id) if not collection.directory: raise ValueError(f"Collection '{collection.name}' has no directory. Cannot reorganize.") @@ -362,9 +359,7 @@ def get_batch_reorganization_preview( """ from sqlalchemy.orm import joinedload - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + collection = Collection.find_by_id(session, collection_id) # Get the current file format from config current_config_format = collection._get_current_file_format() @@ -891,9 +886,7 @@ def execute_reorganization( with get_db_session(commit=True) as session: return execute_reorganization(collection_id, session) - collection = session.query(Collection).filter_by(id=collection_id).one_or_none() - if not collection: - raise UnknownCollection(f"Collection with ID {collection_id} not found") + collection = Collection.find_by_id(session, collection_id) if not collection.directory: raise ValueError(f"Collection '{collection.name}' has no directory. Cannot reorganize.") diff --git a/wrolpi/collections/schema.py b/wrolpi/collections/schema.py index 9a20a487..8e28c650 100644 --- a/wrolpi/collections/schema.py +++ b/wrolpi/collections/schema.py @@ -7,12 +7,45 @@ from dataclasses import dataclass, field from typing import List, Optional +@dataclass +class CollectionCreateRequest: + """Request body for creating a collection (default kind 'playlist').""" + name: str + description: Optional[str] = None + kind: str = 'playlist' + + +@dataclass +class AddItemRequest: + """Request body for adding an item to a collection. + + ``item_kind`` selects which fields are required: + - 'file': file_group_id + - 'zim': zim_id + zim_entry + - 'url': url + """ + item_kind: str = 'file' + file_group_id: Optional[int] = None + zim_id: Optional[int] = None + zim_entry: Optional[str] = None + url: Optional[str] = None + title: Optional[str] = None + position: Optional[int] = None + + +@dataclass +class ReorderItemsRequest: + """Request body for reordering a collection's items by their ids.""" + item_ids: List[int] = field(default_factory=list) + + @dataclass class CollectionUpdateRequest: """Request body for updating a collection.""" directory: Optional[str] = None tag_name: Optional[str] = None description: Optional[str] = None + name: Optional[str] = None @dataclass diff --git a/wrolpi/collections/sync.py b/wrolpi/collections/sync.py new file mode 100644 index 00000000..3a05835a --- /dev/null +++ b/wrolpi/collections/sync.py @@ -0,0 +1,355 @@ +"""On-disk synchronization of playlist collections. + +Each playlist (kind='playlist') is materialized into a per-playlist subdirectory of the configured +playlists directory. A tagged playlist is namespaced under its tag (like channel tags), an untagged +one lives directly under the playlists directory: + + /playlists/// tagged playlist + /playlists// untagged playlist + +Items are ordered by a position prefix within the subdirectory: + + 0001_ hard links of a FileGroup's files (file item) + 0002_.html a stub redirecting to a Zim article (zim item) + 0003_<title>.html a stub redirecting to a URL (url item) + +This gives an offline-browsable, ordered view of the playlist. The playlists directory is excluded +from indexing (see ``WROLPiConfig.import_config``), so these managed files are not re-indexed. + +Only files matching the ``NNNN_`` position prefix are managed here; anything else a user puts in the +directory is left untouched. + +Deletion safety: a managed file is only removed if it is one of our generated HTML stubs, or it still +has another hard link outside the playlists directory (``st_nlink > 1``). We never delete a file +that would lose its last copy -- mirroring the Tags Directory behaviour. +""" +import html +import pathlib +import re +import shutil +import urllib.parse + +from wrolpi.common import get_media_directory, get_wrolpi_config, escape_file_name, logger +from wrolpi.db import get_db_session +from wrolpi.switches import register_switch_handler, ActivateSwitchMethod + +logger = logger.getChild(__name__) + +__all__ = ['sync_playlists_directory', 'get_playlists_directory', 'validate_playlists_destination', + 'cleanup_playlist_directory'] + +# Files we create are prefixed with a zero-padded 1-based position, e.g. "0001_". +MANAGED_NAME = re.compile(r'^\d{4}_') +README_NAME = 'README.txt' +# Embedded in generated stub HTML so we can recognize (and therefore safely delete) our own stubs, +# even though a stub is not a hard link. +STUB_MARKER = '<!--wrolpi-playlist-stub-->' + + +def get_playlists_directory() -> pathlib.Path: + return get_media_directory() / get_wrolpi_config().playlists_destination + + +def validate_playlists_destination(destination: str, config=None) -> str: + """Validate a playlists_destination value, returning an error message (empty string if valid). + + The sync deletes loose files at the destination's root and prunes empty directories beneath it, + so the destination must be a plain relative directory inside the media directory which does not + overlap (equal/ancestor/descendant) any other content destination. + """ + config = config or get_wrolpi_config() + parts = pathlib.PurePosixPath(destination or '').parts + if not parts: + return 'Playlists directory cannot be empty' + if pathlib.PurePosixPath(destination).is_absolute(): + return 'Playlists directory must be relative to media directory' + if '..' in parts or '.' in parts: + return 'Playlists directory cannot contain ".." or "."' + + others = { + 'videos_destination': config.videos_destination, + 'archive_destination': config.archive_destination, + 'map_destination': config.map_destination, + 'zims_destination': config.zims_destination, + 'tags directory': 'tags', # Fixed path, see wrolpi.tags.get_tags_directory. + } + for name, other in others.items(): + if not other: + continue + # Destinations like 'videos/%(channel_tag)s/...' are templates; any directory can match a + # template segment, so only the static prefix is comparable. + other_parts = [] + for part in pathlib.PurePosixPath(other).parts: + if '%(' in part: + break + other_parts.append(part) + if not other_parts: + continue + other_parts = tuple(other_parts) + # Reject equality or any ancestor/descendant relationship: the sync would delete or prune + # files belonging to the other destination. + overlap = min(len(parts), len(other_parts)) + if parts[:overlap] == other_parts[:overlap]: + return f'Playlists directory cannot overlap {name} ({other!r})' + return '' + + +def _position_prefix(position: int) -> str: + return f'{position:04d}_' + + +def _safe_redirect_url(url: str) -> str: + """Neutralize dangerous URL schemes for use in a stub's href/meta-refresh. + + Allows http(s) and scheme-less (relative) URLs like ``/api/...`` or ``/map?...``; anything else + (``javascript:``, ``data:``, ...) becomes ``about:blank`` so opening a stub cannot run script. + """ + url = (url or '').strip() + # Strip control characters that could smuggle a scheme past the check (e.g. "java\tscript:"). + url = ''.join(c for c in url if ord(c) >= 0x20 and c != '\x7f') + scheme = urllib.parse.urlparse(url).scheme.lower() + if scheme and scheme not in ('http', 'https'): + return 'about:blank' + return url + + +def _stub_html(title: str, url: str) -> str: + """A tiny HTML file that redirects to (and links to) a local URL.""" + safe_url = html.escape(_safe_redirect_url(url), quote=True) + safe_title = html.escape(title or url or 'link') + return ( + '<!DOCTYPE html>\n<html lang="en"><head><meta charset="utf-8">' + f'{STUB_MARKER}' + f'<meta http-equiv="refresh" content="0; url={safe_url}">' + f'<title>{safe_title}\n' + f'

Opening {safe_title} …

\n' + ) + + +def _zim_entry_url(zim_id: int, zim_entry: str) -> str: + return f'/api/zim/{zim_id}/entry/{urllib.parse.quote(zim_entry or "")}' + + +def _item_title(item) -> str: + if item.title: + return item.title + if item.item_kind == 'url': + return item.url or 'link' + if item.item_kind == 'zim': + return (item.zim_entry or 'article').split('/')[-1] + return 'item' + + +def _hardlink_or_copy(source: pathlib.Path, link: pathlib.Path): + """Hard-link source to link (replacing a stale link); fall back to copy across filesystems.""" + if link.exists(): + try: + if link.stat().st_ino == source.stat().st_ino: + return # Already the correct hardlink. + except FileNotFoundError: + pass + link.unlink() + link.parent.mkdir(parents=True, exist_ok=True) + try: + link.hardlink_to(source) + except OSError: + # Cross-filesystem (EXDEV) or other link failure -> copy instead. + shutil.copy2(source, link) + + +def _default_playlist_subdir(playlists_directory: pathlib.Path, collection) -> pathlib.Path: + """The managed location for a playlist: tagged playlists are namespaced under their tag + (``//``), untagged playlists live directly under the playlists + directory (``/``). Ignores any explicit custom directory.""" + base = playlists_directory + if collection.tag_name: + base = base / escape_file_name(collection.tag_name) + return base / escape_file_name(collection.name) + + +def _playlist_subdir(playlists_directory: pathlib.Path, collection) -> pathlib.Path: + """The directory a playlist's files live in. + + A playlist with an explicit custom ``directory`` uses it as-is (it may live anywhere in the + media directory); otherwise the managed location is used. + """ + if collection.directory: + return pathlib.Path(collection.directory) + return _default_playlist_subdir(playlists_directory, collection) + + +def _is_stub(path: pathlib.Path) -> bool: + """True if this file is one of our generated HTML stubs (carries STUB_MARKER).""" + try: + with path.open('rb') as fh: + return STUB_MARKER.encode() in fh.read(512) + except OSError: + return False + + +def _safe_unlink_managed(path: pathlib.Path) -> bool: + """Delete a managed playlist file only when it is safe to do so. + + Safe means: it is one of our generated stubs, or it still has another hard link outside the + playlists directory (``st_nlink > 1``). A file that would lose its last copy is never deleted + (we warn instead). Returns True if the file is gone afterward. + """ + try: + st = path.stat() + except FileNotFoundError: + return True + if _is_stub(path) or st.st_nlink > 1: + logger.debug(f'Removing playlist file: {path}') + path.unlink() + return True + logger.warning(f'Refusing to delete playlist file without another hardlink: {path}') + return False + + +def cleanup_playlist_directory(directory: pathlib.Path): + """Clean up a directory a playlist no longer occupies (deleted, moved, or re-pointed). + + Custom playlist directories live outside the playlists root, so the orphan pruning in + ``sync_playlists_directory`` never sees them -- the mutation that abandons the directory calls + this instead. Managed (``NNNN_``) files are removed with the usual stub/hardlink safety, then + the directory is deleted only if it is empty; user files are never touched. + """ + directory = pathlib.Path(directory) + if not directory.is_dir(): + return + for child in list(directory.iterdir()): + if child.is_file() and MANAGED_NAME.match(child.name): + _safe_unlink_managed(child) + if next(directory.iterdir(), None) is None: + logger.debug(f'Removing abandoned playlist directory: {directory}') + directory.rmdir() + else: + logger.warning(f'Keeping abandoned playlist directory with remaining files: {directory}') + + +def _sync_one_playlist(playlists_directory: pathlib.Path, collection) -> pathlib.Path: + """Materialize a single playlist's items into its subdirectory. Returns the subdir.""" + directory = _playlist_subdir(playlists_directory, collection) + directory.mkdir(parents=True, exist_ok=True) + + desired = set() # Names (relative to `directory`) we should have created. + for item in collection.items: # Ordered by position. + prefix = _position_prefix(item.position) + if item.item_kind == 'file' and item.file_group and item.file_group.files: + for source in item.file_group.my_paths(): + if not source.is_file(): + continue + name = f'{prefix}{source.name}' + _hardlink_or_copy(source, directory / name) + desired.add(name) + elif item.item_kind == 'zim': + name = f'{prefix}{escape_file_name(_item_title(item))}.html' + (directory / name).write_text(_stub_html(_item_title(item), + _zim_entry_url(item.zim_id, item.zim_entry))) + desired.add(name) + elif item.item_kind == 'url': + name = f'{prefix}{escape_file_name(_item_title(item))}.html' + (directory / name).write_text(_stub_html(_item_title(item), item.url)) + desired.add(name) + + # Remove our previously-managed files that are no longer wanted (safely; only NNNN_ files). + for path in directory.iterdir(): + if path.is_file() and MANAGED_NAME.match(path.name) and path.name not in desired: + _safe_unlink_managed(path) + + return directory + + +def _write_readme(playlists_directory: pathlib.Path): + playlists_directory.mkdir(parents=True, exist_ok=True) + readme = playlists_directory / README_NAME + if not readme.is_file(): + readme.write_text( + 'This directory contains your playlists as ordered files.\n\n' + 'WARNING: This directory is controlled by WROLPi. Managed files (named like 0001_*)\n' + 'are created and deleted automatically, and any loose file placed directly in this\n' + 'directory will be removed.\n' + ) + + +def _delete_stray_root_files(playlists_directory: pathlib.Path): + """Remove loose files at the playlists root. + + Only the README and playlist (or tag) subdirectories belong at the root of this + WROLPi-managed directory, so any other file a user drops in (e.g. ``playlists/foo``) is deleted. + """ + for path in playlists_directory.iterdir(): + if path.is_file() and path.name != README_NAME: + logger.debug(f'Removing stray file from playlists root: {path}') + path.unlink() + + +def _prune_orphan_dirs(playlists_directory: pathlib.Path, wanted_subdirs): + """Remove playlist directories (and now-empty tag directories) that are no longer wanted. + + Managed files inside an orphaned directory are removed with the same safety as item removal + (stub or another-hardlink only). A directory with surviving files -- because it holds + non-managed files, or files we refused to delete -- is kept. + """ + wanted = set(wanted_subdirs) + # Keep wanted playlist dirs and the tag directories that contain them. + keep = set(wanted) + for directory in wanted: + for parent in directory.parents: + if parent == playlists_directory: + break + keep.add(parent) + + # Deepest first, so a playlist leaf dir is emptied before we try to remove its tag dir. + all_dirs = sorted((p for p in playlists_directory.rglob('*') if p.is_dir()), + key=lambda p: len(p.parts), reverse=True) + for path in all_dirs: + if path in keep: + continue + for child in list(path.iterdir()): + if child.is_file() and MANAGED_NAME.match(child.name): + _safe_unlink_managed(child) + if next(path.iterdir(), None) is None: + logger.debug(f'Removing orphaned playlist directory: {path}') + path.rmdir() + else: + logger.warning(f'Refusing to delete playlist directory with remaining files: {path}') + + +@register_switch_handler('sync_playlists_directory') +def sync_playlists_directory(): + """Synchronize all playlist collections to the playlists directory.""" + from .models import Collection + + # The sync deletes files; refuse to run against a destination that escapes the media directory + # or overlaps another content destination (e.g. a hand-edited wrolpi.yaml). + if error := validate_playlists_destination(get_wrolpi_config().playlists_destination): + message = f'Refusing to sync playlists directory: {error}' + logger.error(message) + raise RuntimeError(message) + + playlists_directory = get_playlists_directory() + media_directory = get_media_directory().resolve() + resolved = playlists_directory.resolve() + if resolved == media_directory or not resolved.is_relative_to(media_directory): + message = f'Refusing to sync playlists directory outside media directory: {resolved}' + logger.error(message) + raise RuntimeError(message) + + with get_db_session() as session: + try: + collections = session.query(Collection).filter_by(kind='playlist').all() + if not collections and not playlists_directory.is_dir(): + # No playlists and no directory: nothing to materialize or clean up. Don't create + # an empty managed directory (with its README) on systems that use no playlists. + return + _write_readme(playlists_directory) + wanted_subdirs = {_sync_one_playlist(playlists_directory, c) for c in collections} + _prune_orphan_dirs(playlists_directory, wanted_subdirs) + _delete_stray_root_files(playlists_directory) + except Exception as e: + logger.error('Failed to sync playlists directory', exc_info=e) + raise + + +sync_playlists_directory: ActivateSwitchMethod diff --git a/wrolpi/collections/test/test_collection_lifecycle.py b/wrolpi/collections/test/test_collection_lifecycle.py index 0c473a21..9ddbf084 100644 --- a/wrolpi/collections/test/test_collection_lifecycle.py +++ b/wrolpi/collections/test/test_collection_lifecycle.py @@ -1,10 +1,8 @@ import pathlib import pytest -import yaml from sqlalchemy.orm import Session -from wrolpi.collections import collections_config from wrolpi.collections.models import Collection from wrolpi.common import get_media_directory @@ -16,11 +14,10 @@ async def test_collection_lifecycle_end_to_end(async_client, test_session: Sessi Lifecycle: - Start with no collections - Create a directory-restricted collection with a unique directory containing some videos - - Dump to config (collections.yaml) - Add a new video to that directory; ensure collection fetches show it - Remove one video from the collection only (files remain) - Delete another video entirely (files removed and item gone from collection) - - Delete the collection; dump config; files remain (except the deleted video) + - Delete the collection; files remain (except the deleted video) """ # 1) Start clean assert test_directory.is_dir() @@ -57,19 +54,7 @@ async def test_collection_lifecycle_end_to_end(async_client, test_session: Sessi fg_ids = {i.file_group_id for i in items} assert {v1.file_group_id, v2.file_group_id, v3.file_group_id}.issubset(fg_ids) - # 4) Dump collections config; verify entry present - collections_config.dump_config() - config_file = collections_config.get_file() - assert config_file.is_file(), f"Expected config file at {config_file}" - - data = yaml.safe_load(config_file.read_text()) - assert isinstance(data, dict) - dumped = data.get('collections', []) - # Config stores relative paths for portability - assert any(c.get('name') == 'Lifecycle Test Collection' and c.get('directory') == str(rel_dir) and c.get( - 'kind') == 'channel' for c in dumped) - - # 5) Add a new video to the same directory; ensure fetch methods include it after population + # 4) Add a new video to the same directory; ensure fetch methods include it after population v4 = video_factory(with_video_file=coll_dir / 'v4.mp4') test_session.commit() @@ -107,12 +92,7 @@ async def test_collection_lifecycle_end_to_end(async_client, test_session: Sessi # 8) Finally, delete the collection test_session.delete(coll) test_session.commit() - - # Dump config again; collection should be removed from config - collections_config.dump_config() - data = yaml.safe_load(config_file.read_text()) - dumped = data.get('collections', []) - assert not any(c.get('name') == 'Lifecycle Test Collection' and c.get('directory') == str(rel_dir) for c in dumped) + assert test_session.query(Collection).count() == 0 # All remaining files should still exist except the one for the deleted video assert v1.file_group.primary_path.exists() diff --git a/wrolpi/collections/test/test_collection_tagging.py b/wrolpi/collections/test/test_collection_tagging.py index 2d71ef4c..16b02cd1 100644 --- a/wrolpi/collections/test/test_collection_tagging.py +++ b/wrolpi/collections/test/test_collection_tagging.py @@ -1,19 +1,14 @@ import pathlib import pytest -import yaml from sqlalchemy.orm import Session -from wrolpi.collections import collections_config from wrolpi.collections.models import Collection from wrolpi.common import get_media_directory -# from wrolpi.switches import await_switches - - @pytest.mark.asyncio -async def test_tagging_directory_collection_moves_files_and_updates_config( +async def test_tagging_directory_collection_moves_files( test_session: Session, test_directory: pathlib.Path, video_factory, @@ -44,15 +39,6 @@ async def test_tagging_directory_collection_moves_files_and_updates_config( # Verify it populated assert {i.file_group_id for i in coll.get_items(test_session)} == {v1.file_group_id, v2.file_group_id} - # Dump config and verify (config stores relative paths for portability) - collections_config.dump_config() - cfg_file = collections_config.get_file() - data = yaml.safe_load(cfg_file.read_text()) - dumped = data.get('collections', []) - assert any( - c.get('name') == 'Funny Clips' and c.get('directory') == str(rel_dir) and c.get('kind') == 'channel' for c in - dumped) - # Tag the collection -> should move to videos// tag = await tag_factory('Funny') coll.set_tag(tag.id) @@ -79,18 +65,12 @@ async def test_tagging_directory_collection_moves_files_and_updates_config( assert str(dest_dir) in str(v1.file_group.primary_path) assert str(dest_dir) in str(v2.file_group.primary_path) - # Dump config again and verify updated directory and tag (config stores relative paths) - collections_config.dump_config() - data = yaml.safe_load(cfg_file.read_text()) - dumped = data.get('collections', []) - rel_dest_dir = dest_dir.relative_to(get_media_directory()) - assert any( - c.get('name') == 'Funny Clips' and c.get('directory') == str(rel_dest_dir) and c.get('tag_name') == 'Funny' for - c in dumped) + # The tag is recorded on the collection. + assert coll.tag_name == 'Funny' @pytest.mark.asyncio -async def test_tagging_unrestricted_collection_updates_config_only( +async def test_tagging_unrestricted_collection_does_not_move_files( test_session: Session, test_directory: pathlib.Path, make_files_structure, @@ -105,44 +85,25 @@ async def test_tagging_unrestricted_collection_updates_config_only( fg1, fg2 = make_files_structure(files, file_groups=True, session=test_session) test_session.commit() - # Create an unrestricted collection and add both files + # Create an unrestricted collection (no directory) and add both files manually. coll = Collection.from_config(test_session, {'name': 'Loose Stuff', 'kind': 'channel'}) - # Unrestricted, so add manually coll.add_file_groups(test_session, [fg1, fg2]) test_session.commit() + assert coll.directory is None - # Dump config and verify directory is absent/None - collections_config.dump_config() - cfg_file = collections_config.get_file() - data = yaml.safe_load(cfg_file.read_text()) - dumped = data.get('collections', []) - entry = next(c for c in dumped if c.get('name') == 'Loose Stuff') - assert 'directory' not in entry - - # Tag the collection; files should not move + # Tag the collection; files should not move (no directory to move into). before_paths = (fg1.primary_path, fg2.primary_path) - # Create a Tag synchronously and assign by name - # from wrolpi.tags import Tag - # tag = Tag(name='Favorites', color='#00ff00') - # test_session.add(tag) - # test_session.commit() await tag_factory('Favorites') coll.set_tag('Favorites') test_session.commit() - # Assert paths unchanged + # Assert paths unchanged and the tag was applied, directory still unset. test_session.refresh(fg1) test_session.refresh(fg2) after_paths = (fg1.primary_path, fg2.primary_path) assert before_paths == after_paths - - # Dump config and verify tag written, directory still not set - collections_config.dump_config() - data = yaml.safe_load(cfg_file.read_text()) - dumped = data.get('collections', []) - entry = next(c for c in dumped if c.get('name') == 'Loose Stuff') - assert entry.get('tag_name') == 'Favorites' - assert 'directory' not in entry + assert coll.tag_name == 'Favorites' + assert coll.directory is None def test_set_tag_on_directory_less_domain_collection(test_session): @@ -221,3 +182,27 @@ async def test_tag_collection_removes_tag_and_updates_directory(test_session, te # Verify result contains correct directory assert result['directory'] == str(untagged_dir.relative_to(test_directory)) + + +@pytest.mark.asyncio +async def test_tag_collection_creates_new_tag(test_session, test_directory, await_switches): + """Tagging a collection with a Tag that does not exist yet creates the Tag and links it.""" + from wrolpi.collections.lib import tag_collection + from wrolpi.tags import Tag + + directory = test_directory / 'archive' / 'example.com' + directory.mkdir(parents=True, exist_ok=True) + collection = Collection(name='example.com', kind='domain', directory=directory) + test_session.add(collection) + test_session.commit() + + assert test_session.query(Tag).filter_by(name='Brand New').one_or_none() is None + + result = await tag_collection(test_session, collection_id=collection.id, tag_name='Brand New') + test_session.commit() + await await_switches(timeout=2) + + tag = test_session.query(Tag).filter_by(name='Brand New').one() + assert collection.tag_id == tag.id + assert collection.tag_name == 'Brand New' + assert result['tag_name'] == 'Brand New' diff --git a/wrolpi/collections/test/test_playlist_items.py b/wrolpi/collections/test/test_playlist_items.py new file mode 100644 index 00000000..b1344283 --- /dev/null +++ b/wrolpi/collections/test/test_playlist_items.py @@ -0,0 +1,390 @@ +"""Tests for playlist CollectionItem operations: create, add file/url items, reorder, remove.""" +from http import HTTPStatus + +import pytest + +from wrolpi.collections.models import Collection + + +@pytest.mark.asyncio +async def test_create_playlist(async_client, test_session): + """A playlist collection can be created via the API.""" + _, response = await async_client.post('/api/collections', json={'name': 'Fire Making'}) + assert response.status_code == HTTPStatus.CREATED, response.json + data = response.json['collection'] + assert data['name'] == 'Fire Making' + assert data['kind'] == 'playlist' + assert data['item_count'] == 0 + assert data['items'] == [] + # Persisted to the database. + assert test_session.query(Collection).filter_by(name='Fire Making', kind='playlist').one() + + +@pytest.mark.asyncio +async def test_create_playlist_rejects_blank_and_duplicate(async_client, test_session): + _, response = await async_client.post('/api/collections', json={'name': ' '}) + assert response.status_code == HTTPStatus.BAD_REQUEST + + _, response = await async_client.post('/api/collections', json={'name': 'Dup'}) + assert response.status_code == HTTPStatus.CREATED + _, response = await async_client.post('/api/collections', json={'name': 'Dup'}) + assert response.status_code == HTTPStatus.BAD_REQUEST + + +@pytest.mark.asyncio +async def test_add_url_item(async_client, test_session): + """A url item (e.g. a map location) can be added and is returned in order.""" + _, response = await async_client.post('/api/collections', json={'name': 'Links'}) + cid = response.json['collection']['id'] + + _, response = await async_client.post(f'/api/collections/{cid}/items', json={ + 'item_kind': 'url', 'url': '/map?lat=40.76&lon=-111.89&z=12', 'title': 'Salt Lake City'}) + assert response.status_code == HTTPStatus.CREATED, response.json + item = response.json['item'] + assert item['item_kind'] == 'url' + assert item['url'] == '/map?lat=40.76&lon=-111.89&z=12' + assert item['title'] == 'Salt Lake City' + assert item['position'] == 1 + + _, response = await async_client.get(f'/api/collections/{cid}') + data = response.json['collection'] + assert data['item_count'] == 1 + assert [i['url'] for i in data['items']] == ['/map?lat=40.76&lon=-111.89&z=12'] + + +@pytest.mark.asyncio +async def test_add_url_item_requires_url(async_client, test_session): + _, response = await async_client.post('/api/collections', json={'name': 'Links'}) + cid = response.json['collection']['id'] + _, response = await async_client.post(f'/api/collections/{cid}/items', json={'item_kind': 'url'}) + assert response.status_code == HTTPStatus.BAD_REQUEST + + +@pytest.mark.asyncio +async def test_add_item_to_unknown_collection(async_client, test_session): + _, response = await async_client.post('/api/collections/99999/items', + json={'item_kind': 'url', 'url': '/x'}) + assert response.status_code == HTTPStatus.NOT_FOUND + + +@pytest.mark.asyncio +async def test_add_file_item(async_client, test_session, test_directory): + """An existing FileGroup can be added as a file item.""" + from wrolpi.files.models import FileGroup + + _, response = await async_client.post('/api/collections', json={'name': 'Mixed'}) + cid = response.json['collection']['id'] + + path = test_directory / 'guide.pdf' + path.write_bytes(b'%PDF-1.4 test') + fg = FileGroup.from_paths(test_session, path) + test_session.commit() + + _, response = await async_client.post(f'/api/collections/{cid}/items', json={ + 'item_kind': 'file', 'file_group_id': fg.id}) + assert response.status_code == HTTPStatus.CREATED, response.json + assert response.json['item']['item_kind'] == 'file' + + _, response = await async_client.get(f'/api/collections/{cid}') + items = response.json['collection']['items'] + assert len(items) == 1 + assert items[0]['file_group']['id'] == fg.id + + +@pytest.mark.asyncio +async def test_reorder_and_remove_items(async_client, test_session): + """Items can be reordered by id and removed, with positions kept contiguous.""" + _, response = await async_client.post('/api/collections', json={'name': 'Order'}) + cid = response.json['collection']['id'] + + ids = [] + for name in ('a', 'b', 'c'): + _, response = await async_client.post(f'/api/collections/{cid}/items', json={ + 'item_kind': 'url', 'url': f'/u/{name}', 'title': name}) + ids.append(response.json['item']['id']) + + # Reverse the order. + _, response = await async_client.put(f'/api/collections/{cid}/items/order', + json={'item_ids': list(reversed(ids))}) + assert response.status_code == HTTPStatus.OK, response.json + items = response.json['collection']['items'] + assert [i['url'] for i in items] == ['/u/c', '/u/b', '/u/a'] + assert [i['position'] for i in items] == [1, 2, 3] + + # Remove the middle item (originally 'b'). + _, response = await async_client.delete(f'/api/collections/{cid}/items/{ids[1]}') + assert response.status_code == HTTPStatus.NO_CONTENT + + _, response = await async_client.get(f'/api/collections/{cid}') + items = response.json['collection']['items'] + assert [i['url'] for i in items] == ['/u/c', '/u/a'] + assert [i['position'] for i in items] == [1, 2] + + +@pytest.mark.asyncio +async def test_tag_playlist_without_directory(async_client, test_session): + """A playlist can be tagged (and untagged) even though it has no directory.""" + _, response = await async_client.post('/api/collections', json={'name': 'Tagged'}) + cid = response.json['collection']['id'] + + # Set a tag -- unlike domains/channels, no directory is required. + _, response = await async_client.put(f'/api/collections/{cid}', json={'tag_name': 'Cooking'}) + assert response.status_code == HTTPStatus.OK, response.json + assert response.json['collection']['tag_name'] == 'Cooking' + + # Clearing the tag works too. + _, response = await async_client.put(f'/api/collections/{cid}', json={'tag_name': ''}) + assert response.status_code == HTTPStatus.OK, response.json + assert response.json['collection']['tag_name'] is None + + +@pytest.mark.asyncio +async def test_playlist_tag_info_suggests_managed_location(async_client, test_session): + """tag_info for a playlist suggests the managed tag location, like channels/domains.""" + _, response = await async_client.post('/api/collections', json={'name': 'Fire Making'}) + cid = response.json['collection']['id'] + + # With a tag: //. + _, response = await async_client.post(f'/api/collections/{cid}/tag_info', + json={'tag_name': 'Survival'}) + assert response.status_code == HTTPStatus.OK, response.json + assert response.json['suggested_directory'] == 'playlists/Survival/Fire Making' + assert response.json['conflict'] is False + + # Without a tag (untagging): /. + _, response = await async_client.post(f'/api/collections/{cid}/tag_info', json={'tag_name': None}) + assert response.status_code == HTTPStatus.OK, response.json + assert response.json['suggested_directory'] == 'playlists/Fire Making' + + +@pytest.mark.asyncio +async def test_playlist_directory_matching_managed_location_is_cleared(async_client, test_session): + """Saving a directory equal to the managed tag location stores None (sync keeps auto-managing).""" + from wrolpi.collections.models import Collection + + _, response = await async_client.post('/api/collections', json={'name': 'Managed'}) + cid = response.json['collection']['id'] + + # Tag + move to the suggested (managed) location in one update, like the tag modal does. + _, response = await async_client.put(f'/api/collections/{cid}', json={ + 'tag_name': 'Survival', 'directory': 'playlists/Survival/Managed'}) + assert response.status_code == HTTPStatus.OK, response.json + assert response.json['collection']['tag_name'] == 'Survival' + # The directory equals the managed location, so it is normalized to None (auto-managed). + collection = test_session.query(Collection).filter_by(id=cid).one() + test_session.refresh(collection) + assert collection.directory is None + + # A genuinely custom directory is kept. + _, response = await async_client.put(f'/api/collections/{cid}', json={ + 'directory': 'custom/spot'}) + assert response.status_code == HTTPStatus.OK, response.json + assert response.json['collection']['directory'] == 'custom/spot' + + +@pytest.mark.asyncio +async def test_rename_playlist(async_client, test_session): + """A playlist can be renamed; blank and duplicate names are rejected.""" + _, response = await async_client.post('/api/collections', json={'name': 'Old Name'}) + cid = response.json['collection']['id'] + + _, response = await async_client.put(f'/api/collections/{cid}', json={'name': 'New Name'}) + assert response.status_code == HTTPStatus.OK, response.json + assert response.json['collection']['name'] == 'New Name' + + # Renaming to another playlist's name is rejected. + _, response = await async_client.post('/api/collections', json={'name': 'Other'}) + assert response.status_code == HTTPStatus.CREATED + _, response = await async_client.put(f'/api/collections/{cid}', json={'name': 'Other'}) + assert response.status_code == HTTPStatus.BAD_REQUEST + + # A blank name is rejected. + _, response = await async_client.put(f'/api/collections/{cid}', json={'name': ' '}) + assert response.status_code == HTTPStatus.BAD_REQUEST + + +@pytest.mark.asyncio +async def test_adding_duplicate_item_is_idempotent(async_client, test_session): + """Re-adding the same url returns the existing item with 200 (not a new 201).""" + _, response = await async_client.post('/api/collections', json={'name': 'Dups'}) + cid = response.json['collection']['id'] + + _, r1 = await async_client.post(f'/api/collections/{cid}/items', + json={'item_kind': 'url', 'url': '/u/x'}) + assert r1.status_code == HTTPStatus.CREATED + first_id = r1.json['item']['id'] + + _, r2 = await async_client.post(f'/api/collections/{cid}/items', + json={'item_kind': 'url', 'url': '/u/x'}) + assert r2.status_code == HTTPStatus.OK # already existed -> not created + assert r2.json['item']['id'] == first_id + + _, rg = await async_client.get(f'/api/collections/{cid}') + assert rg.json['collection']['item_count'] == 1 + + +@pytest.mark.asyncio +async def test_reorder_ignores_duplicate_ids(async_client, test_session): + """Duplicate ids in the reorder list must not leave a position gap.""" + _, response = await async_client.post('/api/collections', json={'name': 'DupOrder'}) + cid = response.json['collection']['id'] + + ids = [] + for name in ('a', 'b', 'c'): + _, r = await async_client.post(f'/api/collections/{cid}/items', + json={'item_kind': 'url', 'url': f'/u/{name}'}) + ids.append(r.json['item']['id']) + + # Pass the first id twice; positions must stay contiguous (1, 2, 3). + _, response = await async_client.put(f'/api/collections/{cid}/items/order', + json={'item_ids': [ids[0], ids[1], ids[0]]}) + assert response.status_code == HTTPStatus.OK, response.json + positions = sorted(i['position'] for i in response.json['collection']['items']) + assert positions == [1, 2, 3], positions + + +@pytest.mark.asyncio +async def test_playlist_directory_sync(async_client, test_session, test_directory): + """Adding items materializes ordered files into the playlists directory; reorder re-prefixes.""" + from wrolpi.files.models import FileGroup + + _, response = await async_client.post('/api/collections', json={'name': 'Sync Test'}) + cid = response.json['collection']['id'] + + # A real file becomes a FileGroup, added as a file item. + path = test_directory / 'lesson.pdf' + path.write_bytes(b'%PDF-1.4 test') + fg = FileGroup.from_paths(test_session, path) + test_session.commit() + _, rf = await async_client.post(f'/api/collections/{cid}/items', + json={'item_kind': 'file', 'file_group_id': fg.id}) + file_item_id = rf.json['item']['id'] + # A url item. + _, ru = await async_client.post(f'/api/collections/{cid}/items', + json={'item_kind': 'url', 'url': '/map?lat=1&lon=2&z=3', 'title': 'Map Spot'}) + url_item_id = ru.json['item']['id'] + + playlist_dir = test_directory / 'playlists' / 'Sync Test' + names = sorted(p.name for p in playlist_dir.iterdir()) + assert any(n.startswith('0001_') and n.endswith('lesson.pdf') for n in names), names + assert any(n.startswith('0002_') and n.endswith('.html') for n in names), names + # The file is a hard link to the source (same inode). + link = next(p for p in playlist_dir.iterdir() if p.name.startswith('0001_')) + assert link.stat().st_ino == path.stat().st_ino + + # Reorder so the url is first; prefixes must follow the new positions. + await async_client.put(f'/api/collections/{cid}/items/order', + json={'item_ids': [url_item_id, file_item_id]}) + names = sorted(p.name for p in playlist_dir.iterdir()) + assert any(n.startswith('0001_') and n.endswith('.html') for n in names), names + assert any(n.startswith('0002_') and n.endswith('lesson.pdf') for n in names), names + + # Removing the url item drops its stub and re-sequences the file to 0001_. + await async_client.delete(f'/api/collections/{cid}/items/{url_item_id}') + names = sorted(p.name for p in playlist_dir.iterdir()) + assert not any(n.endswith('.html') for n in names), names + assert any(n.startswith('0001_') and n.endswith('lesson.pdf') for n in names), names + + +@pytest.mark.asyncio +async def test_url_stub_neutralizes_dangerous_scheme(async_client, test_session, test_directory): + """A javascript: url item must not produce an executable href in its on-disk stub.""" + _, response = await async_client.post('/api/collections', json={'name': 'XSS'}) + cid = response.json['collection']['id'] + await async_client.post(f'/api/collections/{cid}/items', + json={'item_kind': 'url', 'url': 'javascript:alert(1)', 'title': 'evil'}) + + stub = next(p for p in (test_directory / 'playlists' / 'XSS').iterdir() if p.name.endswith('.html')) + content = stub.read_text() + assert 'javascript:' not in content, content + assert 'about:blank' in content + + +@pytest.mark.asyncio +async def test_playlists_config_round_trip(async_client, test_session, test_directory): + """A playlist's items survive a dump -> wipe DB -> import cycle (config is source of truth).""" + from wrolpi.collections.config import playlists_config + from wrolpi.files.models import FileGroup + + _, r = await async_client.post('/api/collections', json={'name': 'Roundtrip'}) + cid = r.json['collection']['id'] + + path = test_directory / 'a.pdf' + path.write_bytes(b'%PDF-1.4 x') + fg = FileGroup.from_paths(test_session, path) + test_session.commit() + await async_client.post(f'/api/collections/{cid}/items', + json={'item_kind': 'file', 'file_group_id': fg.id}) + await async_client.post(f'/api/collections/{cid}/items', + json={'item_kind': 'url', 'url': '/map?x=1', 'title': 'Spot'}) + + # Dump to playlists.yaml, wipe playlists from the DB, then re-import from config. + playlists_config.dump_config() + test_session.query(Collection).filter_by(kind='playlist').delete() + test_session.commit() + assert test_session.query(Collection).filter_by(kind='playlist').count() == 0 + + playlists_config.import_config() + test_session.expire_all() + + collection = test_session.query(Collection).filter_by(name='Roundtrip', kind='playlist').one() + assert [i.item_kind for i in collection.items] == ['file', 'url'] + assert collection.items[0].file_group_id == fg.id + assert collection.items[1].url == '/map?x=1' + assert collection.items[1].title == 'Spot' + assert [i.position for i in collection.items] == [1, 2] + + +@pytest.mark.asyncio +async def test_add_zim_item_end_to_end(async_client, test_session, test_directory): + """A Zim article can be added, renders an on-disk stub, and survives a config round trip.""" + from modules.zim.models import Zim + from wrolpi.collections.config import playlists_config + from wrolpi.vars import PROJECT_DIR + + zim_path = test_directory / 'wikipedia.zim' + zim_path.write_bytes((PROJECT_DIR / 'test/zim.zim').read_bytes()) + zim = Zim.from_paths(test_session, zim_path) + test_session.commit() + + _, response = await async_client.post('/api/collections', json={'name': 'Articles'}) + cid = response.json['collection']['id'] + + # An entry that does not exist in the Zim is rejected. + _, response = await async_client.post(f'/api/collections/{cid}/items', json={ + 'item_kind': 'zim', 'zim_id': zim.id, 'zim_entry': 'nonexistent-entry'}) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.json + + # A real entry is added. + _, response = await async_client.post(f'/api/collections/{cid}/items', json={ + 'item_kind': 'zim', 'zim_id': zim.id, 'zim_entry': 'home', 'title': 'Home Page'}) + assert response.status_code == HTTPStatus.CREATED, response.json + item = response.json['item'] + assert item['item_kind'] == 'zim' + assert item['zim'] == {'id': zim.id, 'entry': 'home', 'path': str(zim_path)} + + # The on-disk stub redirects to the Zim entry API. + stub = test_directory / 'playlists' / 'Articles' / '0001_Home Page.html' + assert stub.is_file() + assert f'/api/zim/{zim.id}/entry/home' in stub.read_text() + + # Config round trip: the zim item is stored by media-relative path and restored after a wipe. + playlists_config.dump_config() + test_session.query(Collection).filter_by(kind='playlist').delete() + test_session.commit() + playlists_config.import_config() + test_session.expire_all() + + collection = test_session.query(Collection).filter_by(name='Articles', kind='playlist').one() + assert [i.item_kind for i in collection.items] == ['zim'] + assert collection.items[0].zim_id == zim.id + assert collection.items[0].zim_entry == 'home' + assert collection.items[0].title == 'Home Page' + + +@pytest.mark.asyncio +async def test_remove_unknown_item(async_client, test_session): + _, response = await async_client.post('/api/collections', json={'name': 'Empty'}) + cid = response.json['collection']['id'] + _, response = await async_client.delete(f'/api/collections/{cid}/items/99999') + assert response.status_code == HTTPStatus.NOT_FOUND diff --git a/wrolpi/collections/test/test_playlist_sync.py b/wrolpi/collections/test/test_playlist_sync.py new file mode 100644 index 00000000..ca5aae59 --- /dev/null +++ b/wrolpi/collections/test/test_playlist_sync.py @@ -0,0 +1,251 @@ +"""Tests for the on-disk playlist sync: tag namespacing, safe deletion, and directory cleanup.""" +import pytest + +from wrolpi.collections.models import Collection +from wrolpi.collections.sync import sync_playlists_directory, get_playlists_directory, \ + validate_playlists_destination +from wrolpi.common import get_wrolpi_config +from wrolpi.tags import Tag + + +def _make_playlist(session, name, tag=None): + collection = Collection(name=name, kind='playlist') + session.add(collection) + session.flush([collection]) + if tag is not None: + collection.tag_id = tag.id + session.flush() + return collection + + +def test_untagged_playlist_lives_at_top_level(test_session, test_directory): + """An untagged playlist syncs to /.""" + _make_playlist(test_session, 'Fire Making') + test_session.commit() + + sync_playlists_directory() + + assert (get_playlists_directory() / 'Fire Making').is_dir() + + +def test_tagged_playlist_namespaced_under_tag(test_session, test_directory): + """A tagged playlist syncs to //.""" + tag = Tag(name='Survival') + test_session.add(tag) + test_session.flush() + _make_playlist(test_session, 'Fire Making', tag=tag) + test_session.commit() + + sync_playlists_directory() + + assert (get_playlists_directory() / 'Survival' / 'Fire Making').is_dir() + assert not (get_playlists_directory() / 'Fire Making').exists() + + +def test_remove_item_deletes_hardlink_keeps_original(test_session, test_directory): + """Removing a file item deletes its hard link from the playlist dir, but keeps the original.""" + from wrolpi.files.models import FileGroup + + source = test_directory / 'guide.pdf' + source.write_bytes(b'%PDF-1.4 test') + file_group = FileGroup.from_paths(test_session, source) + test_session.commit() + + collection = _make_playlist(test_session, 'Docs') + item = collection.add_file_group(file_group, session=test_session) + test_session.commit() + + sync_playlists_directory() + link = get_playlists_directory() / 'Docs' / '0001_guide.pdf' + assert link.is_file(), 'hard link should be created' + assert source.is_file() + + collection.remove_item(test_session, item.id) + test_session.commit() + + sync_playlists_directory() + assert not link.exists(), 'hard link should be removed when the item is removed' + assert source.is_file(), 'the original file must survive' + + +def test_delete_playlist_removes_its_directory(test_session, test_directory): + """Deleting a playlist removes its on-disk directory.""" + collection = _make_playlist(test_session, 'Temp') + collection.add_url(test_session, '/map?lat=1&lon=2&z=3', title='spot') + test_session.commit() + + sync_playlists_directory() + directory = get_playlists_directory() / 'Temp' + assert directory.is_dir() + + # Delete the playlist; the next sync should prune its now-orphaned directory. + test_session.delete(collection) + test_session.commit() + + sync_playlists_directory() + assert not directory.exists() + + +def test_sync_refuses_to_delete_non_hardlinked_file(test_session, test_directory): + """A managed-looking file with no other hard link (and not a stub) is preserved, not deleted.""" + _make_playlist(test_session, 'Safe') + test_session.commit() + sync_playlists_directory() + + directory = get_playlists_directory() / 'Safe' + orphan = directory / '0001_precious.bin' + orphan.write_bytes(b'\x00\x01 irreplaceable') + + # The playlist has no items, so this managed-looking file is "stale" -- but it has no other + # hard link and is not one of our stubs, so it must NOT be deleted. + sync_playlists_directory() + assert orphan.is_file() + + +def test_sync_deletes_stray_root_file(test_session, test_directory): + """A loose file dropped into the playlists root (not README, not a playlist dir) is removed. + + Only README.txt and playlist subdirectories belong at the root of this WROLPi-managed directory. + """ + playlist = _make_playlist(test_session, 'Keeper') + playlist.add_url(test_session, '/u/x', title='x') + test_session.commit() + sync_playlists_directory() + + playlists_directory = get_playlists_directory() + stray = playlists_directory / 'foo' + stray.write_text('junk a user dropped here') + + sync_playlists_directory() + + assert not stray.exists(), 'a stray root file should be deleted during sync' + # The README and real playlist directories are preserved. + assert (playlists_directory / 'README.txt').is_file() + assert (playlists_directory / 'Keeper').is_dir() + + +def test_sync_refuses_dangerous_destination(test_session, test_directory): + """The sync deletes files, so it must refuse a destination that escapes the media directory + or overlaps another content destination (e.g. from a hand-edited wrolpi.yaml).""" + _make_playlist(test_session, 'Any') + test_session.commit() + + config = get_wrolpi_config() + original = config._config['playlists_destination'] + try: + # Traversal outside the media directory. + config._config['playlists_destination'] = '..' + with pytest.raises(RuntimeError): + sync_playlists_directory() + + # The media directory itself. + config._config['playlists_destination'] = '.' + with pytest.raises(RuntimeError): + sync_playlists_directory() + + # Another content destination: syncing here would delete loose files in videos/. + config._config['playlists_destination'] = 'videos' + with pytest.raises(RuntimeError): + sync_playlists_directory() + finally: + config._config['playlists_destination'] = original + + # The default destination still works. + sync_playlists_directory() + assert (get_playlists_directory() / 'Any').is_dir() + + +def test_validate_playlists_destination(test_directory): + """Destination validation rejects traversal and overlap with other content destinations.""" + config = get_wrolpi_config() + + assert validate_playlists_destination('playlists', config) == '' + assert validate_playlists_destination('my/playlists', config) == '' + + assert validate_playlists_destination('', config) + assert validate_playlists_destination('/absolute', config) + assert validate_playlists_destination('../escape', config) + assert validate_playlists_destination('a/../../b', config) + # Equal to, child of, and parent of videos_destination are all rejected. + assert validate_playlists_destination('videos', config) + assert validate_playlists_destination('videos/playlists', config) + assert validate_playlists_destination('zims', config) + assert validate_playlists_destination('map/pins', config) + assert validate_playlists_destination('tags', config) + + +def test_retag_moves_directory_and_prunes_empty_tag_dir(test_session, test_directory): + """Changing a playlist's tag moves its directory and prunes the now-empty old tag directory.""" + tag = Tag(name='Survival') + test_session.add(tag) + test_session.flush() + collection = _make_playlist(test_session, 'Fire Making', tag=tag) + collection.add_url(test_session, '/u/x', title='x') + test_session.commit() + + sync_playlists_directory() + assert (get_playlists_directory() / 'Survival' / 'Fire Making').is_dir() + + # Remove the tag. + collection.tag_id = None + test_session.commit() + + sync_playlists_directory() + assert (get_playlists_directory() / 'Fire Making').is_dir(), 'moved to top level' + assert not (get_playlists_directory() / 'Survival').exists(), 'empty tag dir pruned' + + +def test_custom_directory_playlist(test_session, test_directory): + """A playlist with a custom `directory` syncs there instead of under the playlists root.""" + custom = test_directory / 'survival' / 'fire' + collection = _make_playlist(test_session, 'Fire Making') + collection.directory = custom + collection.add_url(test_session, '/u/x', title='x') + test_session.commit() + + sync_playlists_directory() + + assert (custom / '0001_x.html').is_file() + # No directory is created under the playlists root for this playlist. + assert not (get_playlists_directory() / 'Fire Making').exists() + + +def test_custom_directory_ignores_tag(test_session, test_directory): + """An explicit custom directory wins over tag namespacing.""" + tag = Tag(name='Survival') + test_session.add(tag) + test_session.flush() + custom = test_directory / 'my' / 'spot' + collection = _make_playlist(test_session, 'Fire Making', tag=tag) + collection.directory = custom + collection.add_url(test_session, '/u/x', title='x') + test_session.commit() + + sync_playlists_directory() + + assert (custom / '0001_x.html').is_file() + assert not (get_playlists_directory() / 'Survival').exists() + + +def test_cleanup_playlist_directory(test_session, test_directory): + """cleanup_playlist_directory removes managed files (safely) and the directory if empty.""" + from wrolpi.collections.sync import cleanup_playlist_directory + + custom = test_directory / 'old' / 'spot' + collection = _make_playlist(test_session, 'Mover') + collection.directory = custom + collection.add_url(test_session, '/u/x', title='x') + test_session.commit() + sync_playlists_directory() + assert (custom / '0001_x.html').is_file() + + cleanup_playlist_directory(custom) + assert not custom.exists(), 'managed stub removed and empty dir deleted' + + # A directory holding a non-managed file is kept (only managed files are removed). + custom2 = test_directory / 'old' / 'spot2' + custom2.mkdir(parents=True) + (custom2 / 'keep.txt').write_text('user file') + cleanup_playlist_directory(custom2) + assert (custom2 / 'keep.txt').is_file() + assert custom2.is_dir() diff --git a/wrolpi/collections/test/test_playlists_config.py b/wrolpi/collections/test/test_playlists_config.py new file mode 100644 index 00000000..fcea78e5 --- /dev/null +++ b/wrolpi/collections/test/test_playlists_config.py @@ -0,0 +1,264 @@ +"""Tests for PlaylistsConfig disaster recovery: a missing or empty playlists.yaml must never +delete playlists from the database (mirroring ChannelsConfig's guards).""" +from copy import deepcopy + +import pytest + +from wrolpi.collections.config import get_playlists_config +from wrolpi.collections.models import Collection + + +@pytest.fixture +def playlists_config(async_client): + """The global playlists config, reset to defaults (it is not reset by + initialize_configs_contexts like other configs).""" + config = get_playlists_config() + config._config = deepcopy(config.default_config) + return config + + +def _make_playlist(session, name='Survives'): + collection = Collection(name=name, kind='playlist') + session.add(collection) + collection.add_url(session, '/map?lat=1&lon=2&z=3', title='spot') + session.commit() + return collection + + +@pytest.mark.asyncio +async def test_import_missing_config_preserves_db(test_session, test_directory, playlists_config): + """If playlists.yaml does not exist, importing must NOT delete playlists from the DB. + + A missing config (partial restore, fresh config directory, disk problem) is not the same as + an explicit "no playlists"; deleting here would cascade into the on-disk sync pruning the + playlist directories and the next dump writing an empty config -- total, permanent loss. + """ + _make_playlist(test_session) + assert not playlists_config.get_file().is_file() + + playlists_config.import_config() + + test_session.expire_all() + collection = test_session.query(Collection).filter_by(kind='playlist').one() + assert collection.name == 'Survives' + assert len(collection.items) == 1 + # Nothing to import is a successful import (matches ChannelsConfig). + assert playlists_config.successful_import is True + + +@pytest.mark.asyncio +async def test_import_empty_config_preserves_db(test_session, test_directory, playlists_config): + """A config file with an empty (or absent) playlists list must NOT delete DB playlists. + + Matches ChannelsConfig: an empty list never deletes DB records. Deleting all playlists is + done through the UI/API, which dumps the (empty) config itself. + """ + _make_playlist(test_session) + + file = playlists_config.get_file() + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text('version: 1\nplaylists: []\n') + + playlists_config.import_config() + + test_session.expire_all() + collection = test_session.query(Collection).filter_by(kind='playlist').one() + assert collection.name == 'Survives' + assert playlists_config.successful_import is True + + +@pytest.mark.asyncio +async def test_db_rebuild_round_trip(test_session, test_directory, playlists_config): + """After a full DB rebuild, playlists are recovered from playlists.yaml. + + File items can only be re-linked after the file refresh has re-indexed their FileGroups; until + then they are skipped with a warning, and restored by a later re-import (the config file still + holds them as media-relative paths). + """ + from wrolpi.files.models import FileGroup + + pdf = test_directory / 'guide.pdf' + pdf.write_bytes(b'%PDF-1.4 x') + file_group = FileGroup.from_paths(test_session, pdf) + collection = Collection(name='Rebuild', kind='playlist') + test_session.add(collection) + test_session.flush([collection]) + collection.add_file_group(file_group, session=test_session) + collection.add_url(test_session, '/map?x=1', title='Spot') + test_session.commit() + + playlists_config.dump_config() + + # Simulate a DB rebuild: collections AND file groups are gone. + test_session.query(Collection).filter_by(kind='playlist').delete() + test_session.query(FileGroup).delete() + test_session.commit() + + playlists_config.import_config() + test_session.expire_all() + collection = test_session.query(Collection).filter_by(name='Rebuild', kind='playlist').one() + # The file is not indexed yet, so only the url item could be restored. + assert [i.item_kind for i in collection.items] == ['url'] + + # The file refresh re-indexes the file; a re-import restores the file item in its position. + FileGroup.from_paths(test_session, pdf) + test_session.commit() + playlists_config.import_config() + test_session.expire_all() + collection = test_session.query(Collection).filter_by(name='Rebuild', kind='playlist').one() + assert [i.item_kind for i in collection.items] == ['file', 'url'] + assert [i.position for i in collection.items] == [1, 2] + + +@pytest.mark.asyncio +async def test_global_refresh_restores_skipped_playlist_items( + test_session, test_directory, playlists_config, refresh_files, await_switches): + """A global refresh re-imports playlists.yaml, restoring items skipped at startup. + + After a DB rebuild, playlists.yaml is imported before any files are indexed, so file items are + skipped. When the global refresh completes, the worker re-imports the config and the skipped + items are restored. + """ + pdf = test_directory / 'guide.pdf' + pdf.write_bytes(b'%PDF-1.4 x') + + file = playlists_config.get_file() + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text( + 'version: 1\n' + 'playlists:\n' + ' - name: Rebuilt\n' + ' items:\n' + ' - file: guide.pdf\n' + ' - url: /map?x=1\n' + ) + + # Startup import: the file is not indexed yet, so only the url item is restored. + playlists_config.import_config() + test_session.expire_all() + collection = test_session.query(Collection).filter_by(name='Rebuilt', kind='playlist').one() + assert [i.item_kind for i in collection.items] == ['url'] + + # A global refresh indexes the file and triggers the re-import. + await refresh_files() + await await_switches() + test_session.expire_all() + + collection = test_session.query(Collection).filter_by(name='Rebuilt', kind='playlist').one() + assert [i.item_kind for i in collection.items] == ['file', 'url'] + assert [i.position for i in collection.items] == [1, 2] + + +@pytest.mark.asyncio +async def test_backup_preview_and_import_merge(test_session, test_directory, playlists_config): + """A backup can be previewed and merged: backup-only playlists are added, current ones win.""" + file = playlists_config.get_file() + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text( + 'version: 3\n' + 'playlists:\n' + ' - name: Current\n' + ' items:\n' + ' - url: /u/current\n' + ) + backup = playlists_config._get_backup_file('20260101') + backup.parent.mkdir(parents=True, exist_ok=True) + backup.write_text( + 'version: 1\n' + 'playlists:\n' + ' - name: Current\n' + ' items:\n' + ' - url: /u/old\n' + ' - name: Restored\n' + ' items:\n' + ' - url: /u/restored\n' + ' - url: /u/restored2\n' + ) + + preview = playlists_config.preview_backup_import('20260101', 'merge') + assert preview['add'] == [dict(type='playlist', name='Restored', items=2)] + assert preview['remove'] == [] + assert preview['unchanged'] == 1 + + playlists_config.import_backup('20260101', 'merge') + test_session.expire_all() + by_name = {c.name: c for c in test_session.query(Collection).filter_by(kind='playlist').all()} + assert sorted(by_name) == ['Current', 'Restored'] + # Merge keeps the current config's version of 'Current'. + assert [i.url for i in by_name['Current'].items] == ['/u/current'] + assert [i.url for i in by_name['Restored'].items] == ['/u/restored', '/u/restored2'] + + +@pytest.mark.asyncio +async def test_backup_preview_and_import_overwrite(test_session, test_directory, playlists_config): + """Overwrite mode replaces the config (and DB) with the backup's playlists.""" + _make_playlist(test_session, name='CurrentOnly') + playlists_config.dump_config() + + backup = playlists_config._get_backup_file('20260101') + backup.parent.mkdir(parents=True, exist_ok=True) + backup.write_text( + 'version: 1\n' + 'playlists:\n' + ' - name: FromBackup\n' + ' items:\n' + ' - url: /u/backup\n' + ) + + preview = playlists_config.preview_backup_import('20260101', 'overwrite') + assert preview['add'] == [dict(type='playlist', name='FromBackup', items=1)] + assert preview['remove'] == [dict(type='playlist', name='CurrentOnly')] + + playlists_config.import_backup('20260101', 'overwrite') + test_session.expire_all() + names = [c.name for c in test_session.query(Collection).filter_by(kind='playlist').all()] + assert names == ['FromBackup'] + + +@pytest.mark.asyncio +async def test_import_still_deletes_playlists_removed_from_config( + test_session, test_directory, playlists_config): + """A non-empty config remains authoritative: playlists absent from it are deleted.""" + _make_playlist(test_session, name='Kept') + _make_playlist(test_session, name='Removed') + + file = playlists_config.get_file() + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text( + 'version: 1\n' + 'playlists:\n' + ' - name: Kept\n' + ' items:\n' + ' - url: /map?lat=1&lon=2&z=3\n' + ) + + playlists_config.import_config() + + test_session.expire_all() + names = [c.name for c in test_session.query(Collection).filter_by(kind='playlist').all()] + assert names == ['Kept'] + + +@pytest.mark.asyncio +async def test_custom_directory_round_trips(test_session, test_directory, playlists_config): + """A playlist's custom directory survives a dump -> wipe -> import cycle.""" + custom = test_directory / 'survival' / 'fire' + collection = _make_playlist(test_session, name='Custom Dir') + collection.directory = custom + test_session.commit() + + playlists_config.dump_config() + + # The config stores the directory relative to the media directory. + data = playlists_config.read_config_file() + entry = next(p for p in data['playlists'] if p['name'] == 'Custom Dir') + assert entry['directory'] == 'survival/fire' + + # Wipe and re-import; the directory is restored. + test_session.query(Collection).filter_by(kind='playlist').delete() + test_session.commit() + playlists_config.import_config() + + test_session.expire_all() + collection = test_session.query(Collection).filter_by(name='Custom Dir').one() + assert str(collection.directory) == str(custom) diff --git a/wrolpi/collections/types.py b/wrolpi/collections/types.py index 7ecc12fb..151f59d4 100644 --- a/wrolpi/collections/types.py +++ b/wrolpi/collections/types.py @@ -135,3 +135,9 @@ collection_type_registry.register( validate_channel_name, 'Channel name must be a non-empty string' ) + +collection_type_registry.register( + 'playlist', + validate_channel_name, # Permissive: any non-empty name. + 'Playlist name must be a non-empty string' +) diff --git a/wrolpi/common.py b/wrolpi/common.py index 355a7466..95b2ba68 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -456,7 +456,7 @@ def get_media_directory() -> Path: DB_CONFIG_FILE_NAMES = {'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml', 'inventories.yaml', - 'collections.yaml'} + 'playlists.yaml'} class ConfigFile: @@ -809,6 +809,7 @@ class WROLPiConfigValidator: map_default_location: dict = None map_destination: str = None nav_color: str = None + playlists_destination: str = None require_cookies_unlocked: bool = None require_media_mounted: bool = None save_ffprobe_json: bool = None @@ -858,14 +859,14 @@ def get_all_configs() -> Dict[str, ConfigFile]: if download_cache_config := get_download_cache_config(): all_configs[download_cache_config.file_name] = download_cache_config - from wrolpi.collections.config import collections_config as _collections_config - if _collections_config: - all_configs[_collections_config.file_name] = _collections_config - from modules.map.pins import get_map_pins_config if map_pins_config := get_map_pins_config(): all_configs[map_pins_config.file_name] = map_pins_config + from wrolpi.collections.config import get_playlists_config + if playlists_config := get_playlists_config(): + all_configs[playlists_config.file_name] = playlists_config + return all_configs @@ -938,15 +939,18 @@ async def import_all_db_configs() -> dict[str, bool]: logger.warning(f'Failed to import inventories config: {e}') results['inventories'] = False - # Collections (uses tags for tag_name) + # Playlists (kind='playlist' collections; uses tags for tag_name). + # NOTE: there is no generic collections config — channels/domains are owned by their own configs + # (channels.yaml/domains.yaml), and authors/subjects are re-derived on refresh. Playlists are + # user-curated with no other config home, so they own playlists.yaml. try: - from wrolpi.collections.config import collections_config as _collections_config - _collections_config.import_config() - results['collections'] = True - logger.debug('collections config imported') + from wrolpi.collections.config import playlists_config + playlists_config.import_config() + results['playlists'] = True + logger.debug('playlists config imported') except Exception as e: - logger.warning(f'Failed to import collections config: {e}') - results['collections'] = False + logger.warning(f'Failed to import playlists config: {e}') + results['playlists'] = False # Map pins (YAML-only, no DB) try: @@ -989,6 +993,7 @@ class WROLPiConfig(ConfigFile): map_default_location=None, map_destination='map', nav_color='violet', + playlists_destination='playlists', require_cookies_unlocked=True, require_media_mounted=True, save_ffprobe_json=True, @@ -1033,6 +1038,14 @@ class WROLPiConfig(ConfigFile): self._config['map_destination'] = self.map_destination or self.default_config['map_destination'] self._config['videos_destination'] = self.videos_destination or self.default_config['videos_destination'] self._config['zims_destination'] = self.zims_destination or self.default_config['zims_destination'] + self._config['playlists_destination'] = \ + self.playlists_destination or self.default_config['playlists_destination'] + + # The playlists directory holds WROLPi-managed hardlinks/stubs; never index it. + ignored = list(self._config.get('ignored_directories') or []) + if self._config['playlists_destination'] not in ignored: + ignored.append(self._config['playlists_destination']) + self._config['ignored_directories'] = ignored self.successful_import = True except Exception as e: @@ -1214,6 +1227,14 @@ class WROLPiConfig(ConfigFile): def archive_destination(self, value: str): self.update({'archive_destination': value}) + @property + def playlists_destination(self) -> str: + return self._config['playlists_destination'] + + @playlists_destination.setter + def playlists_destination(self, value: str): + self.update({'playlists_destination': value}) + @property def map_default_location(self) -> dict: return self._config.get('map_default_location') diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index d0f38d29..14659a14 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -32,7 +32,6 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.domains_config = manager.dict() app.shared_ctx.archive_downloader_config = manager.dict() app.shared_ctx.download_cache_config = manager.dict() - app.shared_ctx.collections_config = manager.dict() app.shared_ctx.map_pins_config = manager.dict() # Shared dicts. app.shared_ctx.refresh = manager.dict() @@ -105,7 +104,6 @@ def reset_shared_contexts(app: Sanic): app.shared_ctx.domains_config.clear() app.shared_ctx.archive_downloader_config.clear() app.shared_ctx.download_cache_config.clear() - app.shared_ctx.collections_config.clear() # Shared dicts. app.shared_ctx.refresh.clear() app.shared_ctx.uploaded_files.clear() @@ -265,12 +263,6 @@ def initialize_configs_contexts(app: Sanic): except Exception as e: logger.error(f'Failed to initialize in-memory download cache config: {e}') - try: - from wrolpi.collections.config import collections_config as _collections_config - _collections_config.initialize(app.shared_ctx.collections_config) - except Exception as e: - logger.error(f'Failed to initialize in-memory collections config: {e}') - try: from modules.map.pins import MAP_PINS_CONFIG MAP_PINS_CONFIG.initialize(app.shared_ctx.map_pins_config) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index c68fd577..1fc73787 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -376,16 +376,14 @@ class Download(ModelHelper, Base): # noqa session.delete(self) session.commit() - # Save appropriate config based on collection kind + # Save appropriate config based on collection kind. Channels/domains own their configs; + # other kinds have no generic collections config to save. if collection_kind == 'channel': from modules.videos.lib import save_channels_config save_channels_config.activate_switch() elif collection_kind == 'domain': from modules.archive.lib import save_domains_config save_domains_config.activate_switch() - elif collection_kind: - from wrolpi.collections.config import save_collections_config - save_collections_config.activate_switch() # Save download config again because this download is now removed from the download lists. save_downloads_config.activate_switch() diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index 69373c38..ba751948 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -1241,6 +1241,10 @@ class FileWorker: if is_global_refresh: flags.refresh_complete.set() flags.global_refresh_active.clear() + # After a DB rebuild, playlists.yaml was imported before any files were indexed, so its + # file/zim items were skipped. They are indexed now; re-import to restore them. + from wrolpi.collections.config import import_playlists_config + import_playlists_config.activate_switch() # Mark job as complete if tracking self._complete_job(task.job_id) diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index cd634119..160bd698 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -176,6 +176,7 @@ def get_settings(_: Request): 'map_destination': wrolpi_config.map_destination, 'nav_color': wrolpi_config.nav_color, 'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative. + 'playlists_destination': wrolpi_config.playlists_destination, 'tags_directory': wrolpi_config.tags_directory, 'throttle_on_startup': wrolpi_config.throttle_on_startup, 'version': __version__, @@ -240,6 +241,23 @@ async def update_settings(_: Request, body: schema.SettingsRequest): elif not body.zims_destination: new_config['zims_destination'] = wrolpi_config.default_config['zims_destination'] + if body.playlists_destination: + # The playlists sync deletes files in its directory, so this destination gets stricter + # validation: no traversal, and no overlap with the other content destinations. Compare + # against the destinations in this same request when provided. + from types import SimpleNamespace + from wrolpi.collections.sync import validate_playlists_destination + effective = SimpleNamespace( + videos_destination=body.videos_destination or wrolpi_config.videos_destination, + archive_destination=body.archive_destination or wrolpi_config.archive_destination, + map_destination=body.map_destination or wrolpi_config.map_destination, + zims_destination=body.zims_destination or wrolpi_config.zims_destination, + ) + if error := validate_playlists_destination(body.playlists_destination, effective): + raise InvalidConfig(error) + else: + new_config['playlists_destination'] = wrolpi_config.default_config['playlists_destination'] + # Validate timezone if provided; empty string clears it. if body.timezone is not None: if body.timezone == '': diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 033d0753..e4128a8a 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -40,6 +40,7 @@ class SettingsResponse: wrol_mode: bool archive_destination: str = 'archive/%(domain_tag)s/%(domain)s' map_destination: str = 'map' + playlists_destination: str = 'playlists' videos_destination: str = 'videos/%(channel_tag)s/%(channel_name)s' zims_destination: str = 'zims' @@ -78,6 +79,7 @@ class SettingsRequest: map_destination: Optional[str] = None nav_color: Optional[str] = None media_directory: Optional[str] = None + playlists_destination: Optional[str] = None tags_directory: Optional[bool] = None throttle_on: Optional[bool] = None throttle_on_startup: Optional[bool] = None diff --git a/wrolpi/tags.py b/wrolpi/tags.py index 48b17861..c74e660d 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -252,6 +252,28 @@ class Tag(ModelHelper, Base): save_tags_config.activate_switch() sync_tags_directory.activate_switch() + @staticmethod + def get_or_create_tag(session: Session, name: str = None, id_: int = None) -> 'Tag': + """Gets or creates Tag object. + + @warning: does not commit""" + if name and id_: + raise RuntimeError('Tag name and id cannot be used together') + + if tag := session.query(Tag).filter(Tag.id == id_).one_or_none(): + return tag + if tag := session.query(Tag).filter(Tag.name == name).one_or_none(): + return tag + + if not name: + raise InvalidTag('Tag name cannot be empty') + + tag = Tag(name=name) + session.add(tag) + # Flush so the new Tag has an id (callers use `.id` immediately). + tag.flush(session) + return tag + @event.listens_for(Tag, 'after_insert') @event.listens_for(Tag, 'after_update') diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 03bbe6df..fd938686 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -798,6 +798,37 @@ async def test_settings_special_directories(async_client, test_wrolpi_config): assert get_wrolpi_config().archive_destination == 'archive/%(domain_tag)s/%(domain)s' +@pytest.mark.asyncio +async def test_settings_playlists_destination_validation(async_client, test_wrolpi_config): + """The playlists destination is strictly validated: the sync deletes files there, so traversal + and overlap with other content destinations are rejected.""" + # A valid change works. + data = {'playlists_destination': 'my_playlists'} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + assert get_wrolpi_config().playlists_destination == 'my_playlists' + + for invalid in ( + '/absolute/not/allowed', + '../outside-media', + 'a/../../b', + 'videos', # equals/overlaps videos_destination's base + 'videos/playlists', # inside videos_destination + 'zims', + 'tags', + ): + data = {'playlists_destination': invalid} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST, \ + f'{invalid!r} should have been rejected' + + # Empty restores the default. + data = {'playlists_destination': ''} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + assert get_wrolpi_config().playlists_destination == 'playlists' + + @pytest.mark.asyncio async def test_settings_save_ffprobe_json(async_client, test_wrolpi_config): """Maintainer can toggle save_ffprobe_json setting."""