rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 02:48:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Merge branch 'master' into release

Browse source
This commit is contained in:
Roland 2026-07-04 09:25:52 -06:00
commit 7d6a77a54e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
55 changed files with 1507 additions and 183 deletions
Download patch file Download diff file Expand all files Collapse all files

2
INSTALL.md
View file

@ -25,7 +25,7 @@ You will need:
Steps:
1. Download the latest `WROLPi-v*-amd64.iso` from [wrolpi.org](https://wrolpi.org).
2. Flash the whole drive with the ISO using Etcher, Rufus, Raspberry Pi Imager, or `dd`:
2. Flash the whole drive with the ISO using Raspberry Pi Imager, Rufus, or `dd`:
```
sudo dd if=WROLPi-v<version>-amd64.iso of=/dev/sdX bs=4M status=progress conv=fsync
```

43
alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py Normal file
View file

@ -0,0 +1,43 @@
"""file_group_textsearch_abc
Add the `textsearch_abc` generated tsvector column to `file_group`. It indexes only a_text (title),
b_text (author), and c_text (description) — excluding d_text (captions/document body), which is ~90% of all
text data. Searches default to this much smaller column; the existing `textsearch` column remains for
opt-in "deep" search.
Adding a STORED generated column rewrites the table; expect a few minutes on large libraries.
Revision ID: e5f7a9c1d3b6
Revises: d4e6f8a0c2b5
Create Date: 2026-07-03
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = 'e5f7a9c1d3b6'
down_revision = 'd4e6f8a0c2b5'
branch_labels = None
depends_on = None
def upgrade():
conn = op.get_bind()
conn.execute(sa.text('''
ALTER TABLE file_group
ADD COLUMN textsearch_abc tsvector
GENERATED ALWAYS AS (
setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char")
) STORED
'''))
conn.execute(sa.text(
'CREATE INDEX IF NOT EXISTS file_group_textsearch_abc_idx ON file_group USING GIN(textsearch_abc)'))
def downgrade():
conn = op.get_bind()
conn.execute(sa.text('DROP INDEX IF EXISTS file_group_textsearch_abc_idx'))
conn.execute(sa.text('ALTER TABLE file_group DROP COLUMN IF EXISTS textsearch_abc'))

23
app/src/App.css
View file

@ -454,10 +454,33 @@ tr.drag-deselecting td {
-ms-user-select: none;
}
/* Filter input and collapse-all button: right-aligned on desktop, full width on mobile (below). */
.file-browser-filter-container {
display: flex;
justify-content: flex-end;
margin-bottom: 0.5em;
}
/* Double the default input width on desktop. */
.ui.input.file-browser-filter {
width: 34em;
max-width: 100%;
}
@media only screen and (max-width: 700px) {
.hide-on-mobile {
display: none !important;
}
.file-browser-filter-container {
justify-content: flex-start;
}
/* Fill the row, but stay on one line with the collapse-all button. */
.ui.input.file-browser-filter {
flex: 1 1 auto;
width: auto;
}
}
/* Ensure step icons are always visible, even when disabled */

2
app/src/DashboardPage.js
View file

@ -469,7 +469,7 @@ export function DashboardPage() {
}
return <div style={{display: 'flex', alignItems: 'center', gap: '0.5em', marginBottom: '2em'}}>
{getSearchResultsInput(inputProps)}
<SearchFilterButton fileFilterOptions={fileMimetypeFilterOptions} showDates={true}
<SearchFilterButton fileFilterOptions={fileMimetypeFilterOptions} showDates={true} showDeep={true}
size={big ? 'big' : undefined}/>
</div>;
};

21
app/src/api.js
View file

@ -187,7 +187,7 @@ export async function tagChannelInfo(channelId, tagName) {
});
}
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline, censored) {
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline, censored, deep) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@ -208,6 +208,9 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
if (censored) {
body['censored'] = true;
}
if (deep) {
body['deep'] = true;
}
console.debug('searching videos', body);
@ -697,7 +700,7 @@ export async function saveCatalog(items) {
}
export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline) {
export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline, deep) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@ -717,6 +720,9 @@ export async function searchArchives(offset, limit, domain, searchStr, order, ta
if (headline) {
body['headline'] = true;
}
if (deep) {
body['deep'] = true;
}
console.debug('searching archives', body);
const response = await apiPost(`${ARCHIVES_API}/search`, body);
@ -1212,7 +1218,7 @@ export async function deleteDownload(downloadId) {
}
}
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order, suffix, path) {
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order, suffix, path, deep) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit), any_tag: anyTag};
if (suffix) {
body['suffix'] = suffix;
@ -1244,6 +1250,9 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta
if (order) {
body['order'] = order;
}
if (deep) {
body['deep'] = true;
}
console.info('searching files', body);
const response = await apiPost(`${API_URI}/files/search`, body);
@ -1988,6 +1997,7 @@ export async function searchEstimateFiles(search_str, tagNames, mimetypes, month
return {
fileGroups: content.file_groups,
fileGroupsDeep: content.file_groups_deep,
}
} else {
console.error('Failed to get file search estimates!');
@ -2220,7 +2230,7 @@ export async function getDoc(fileGroupId) {
}
export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject, mimetype) {
export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject, mimetype, deep) {
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
let body = {offset, limit};
@ -2242,6 +2252,9 @@ export async function searchDocs(offset, limit, searchStr, order, tagNames, auth
if (mimetype) {
body['mimetype'] = mimetype;
}
if (deep) {
body['deep'] = true;
}
console.debug('searching docs', body);
const response = await apiPost(`${DOCS_API}/search`, body);

12
app/src/api/controller.js
View file

@ -75,6 +75,18 @@ export async function getHotspotStatus() {
return controllerFetch('/hotspot/status');
}
export async function getHotspotDevices() {
return controllerFetch('/hotspot/devices');
}
export async function getHotspotSettings() {
return controllerFetch('/hotspot/settings');
}
export async function updateHotspotSettings(settings) {
return controllerFetch('/hotspot/settings', {method: 'POST', body: JSON.stringify(settings)});
}
export async function enableHotspot() {
return controllerFetch('/hotspot/enable', {method: 'POST'});
}

4
app/src/components/Archive.js
View file

@ -77,7 +77,7 @@ import {
useSearchDomain,
useSearchOrder
} from "../hooks/customHooks";
import {FileCards, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
import {DeepSearchHint, FileCards, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import _ from "lodash";
import {ThemeContext} from "../contexts/contexts";
@ -1259,8 +1259,10 @@ function ArchivesPage() {
inputRef={searchInputRef}
viewButton={viewButton}
sorts={archiveOrders}
showDeep={true}
/>
{body}
<DeepSearchHint searchStr={searchStr} files={archives}/>
{paginator}
<TaggedDeleteConfirmModal
open={taggedFileGroups !== null}

4
app/src/components/Docs.js
View file

@ -21,7 +21,7 @@ import {
import {Button, darkTheme, Header, Icon, Segment, Statistic, Tab} from "./Theme";
import {BulkTagModal} from "./BulkTagModal";
import {TaggedDeleteConfirmModal} from "./TaggedDeleteConfirmModal";
import {docMimetypeFilterOptions, FilesView, SearchControlBar} from "./Files";
import {DeepSearchHint, docMimetypeFilterOptions, FilesView, SearchControlBar} from "./Files";
import {useAuthors, useDoc, useOneQuery, useSearchDocs, useSubjects} from "../hooks/customHooks";
import {TagsSelector} from "../Tags";
import {toast} from "react-semantic-toasts-2";
@ -138,8 +138,10 @@ function DocsPage() {
viewButton={viewButton}
sorts={docOrders}
fileFilterOptions={docMimetypeFilterOptions}
showDeep={true}
/>
{body}
<DeepSearchHint searchStr={searchStr} files={docs}/>
{paginator}
<TaggedDeleteConfirmModal
open={taggedFileGroups !== null}

87
app/src/components/FileBrowser.js
View file

@ -24,6 +24,7 @@ import {
} from "./Common";
import React, {useContext, useEffect, useState} from "react";
import {useDropzone} from "react-dropzone";
import {useHotkeys} from "react-hotkeys-hook";
import {deleteFile, ignoreDirectory, makeDirectory, movePaths, renamePath, unignoreDirectory} from "../api";
import _ from 'lodash';
import {SortableTable} from "./SortableTable";
@ -67,6 +68,32 @@ export function splitPathParentAndName(path) {
return [path.substring(0, slashIndex), path.substring(slashIndex + 1, path.length)];
}
export function filterBrowseFiles(browseFiles, filterStr) {
// Reduce the file tree to the rows whose name matches `filterStr`, keeping the ancestor
// folders of every match. A folder whose own name matches keeps its entire subtree.
if (!filterStr) {
return browseFiles;
}
const needle = filterStr.toLowerCase();
const filterList = (files) => {
// `files` may be a dict keyed by name (from the API) or an array.
const fileList = Array.isArray(files) ? files : Object.values(files);
const matches = [];
for (const file of fileList) {
if (pathName(file.path).toLowerCase().includes(needle)) {
matches.push(file);
} else if (file.children) {
const children = filterList(file.children);
if (children.length > 0) {
matches.push({...file, children});
}
}
}
return matches;
};
return filterList(browseFiles);
}
function Folder({folder, onFolderClick, sortData, onFileClick, disabled, loadingFolders}) {
const {handleDragStart, handleDragMove, getDragState, shouldAllowClick, selectedPaths, onSelect} = useContext(DragSelectionContext);
// Creates a single table row for a folder, or a row for itself and indented rows for its children.
@ -219,6 +246,17 @@ export function FileBrowser() {
const pendingAutoSelectRef = React.useRef(null);
// Tagged file groups that would be deleted - shown in confirmation modal
const [taggedFileGroups, setTaggedFileGroups] = React.useState(null);
// Reduces the visible rows to those matching, and their parent folders.
const [filterStr, setFilterStr] = React.useState('');
const filterInputRef = React.useRef();
// 'f' focuses the filter input, like the search inputs on other pages.
useHotkeys('f', (e) => {
e.preventDefault();
if (filterInputRef.current) {
filterInputRef.current.focus();
}
}, {enableOnFormTags: false});
const {settings, fetchSettings} = React.useContext(SettingsContext);
const {inverted} = React.useContext(ThemeContext);
@ -457,6 +495,17 @@ export function FileBrowser() {
/>
</div>;
const onCollapseAll = () => {
if (!openFolders) return;
// Deselect any paths that will no longer be visible (descendants of the open folders).
const newSelectedPaths = selectedPaths.filter(p =>
!openFolders.some(folder => p.startsWith(folder) && p !== folder));
if (newSelectedPaths.length !== selectedPaths.length) {
setSelectedPaths(newSelectedPaths);
}
setOpenFolders(null);
}
const onFolderClick = async (folder) => {
let newFolders;
if (openFolders.indexOf(folder) >= 0) {
@ -481,6 +530,11 @@ export function FileBrowser() {
setOpenFolders(newFolders);
}
const filteredFiles = React.useMemo(
() => browseFiles ? filterBrowseFiles(browseFiles, filterStr) : browseFiles,
[browseFiles, filterStr],
);
if (browseFiles === null) {
return <Placeholder>
<PlaceholderLine/>
@ -493,11 +547,40 @@ export function FileBrowser() {
return <DragSelectionProvider
selectedPaths={selectedPaths}
setSelectedPaths={setSelectedPaths}
browseFiles={browseFiles}
browseFiles={filteredFiles}
openFolders={openFolders}
>
<div className='file-browser-filter-container'>
<Input
icon='filter'
iconPosition='left'
placeholder='Filter files...'
value={filterStr}
onChange={(e, {value}) => setFilterStr(value)}
aria-label='Filter files'
className='file-browser-filter'
ref={filterInputRef}
action={<Button
icon='close'
type='button'
onClick={() => setFilterStr('')}
disabled={!filterStr}
className='search-clear'
aria-label='Clear filter'
/>}
/>
<Button
icon='angle double up'
type='button'
onClick={onCollapseAll}
disabled={!openFolders || openFolders.length === 0}
title='Close all folders'
aria-label='Close all folders'
style={{marginLeft: '0.5em'}}
/>
</div>
<FileBrowserContent
browseFiles={browseFiles}
browseFiles={filteredFiles}
headers={headers}
onFolderClick={onFolderClick}
setPreviewFile={setPreviewFile}

289
app/src/components/FileBrowser.test.js
View file

@ -1,6 +1,6 @@
import React from 'react';
import {act, fireEvent, screen, waitFor} from '@testing-library/react';
import {FileBrowser} from './FileBrowser';
import {FileBrowser, filterBrowseFiles} from './FileBrowser';
import {renderWithProviders} from '../test-utils';
import {SettingsContext} from '../contexts/contexts';
import {FilePreviewContext} from './FilePreview';
@ -478,6 +478,293 @@ describe('FileBrowser', () => {
});
});
describe('Filter', () => {
// foo/ and bar/ are open, their children are loaded.
const nestedBrowseFiles = [
{
path: 'foo/', is_empty: false, children: [
{path: 'foo/foo1.txt', size: 100},
{path: 'foo/foo2.txt', size: 200},
]
},
{
path: 'bar/', is_empty: false, children: [
{path: 'bar/bar.1.txt', size: 300},
]
},
];
describe('filterBrowseFiles', () => {
it('returns files unchanged when filter is empty', () => {
expect(filterBrowseFiles(nestedBrowseFiles, '')).toBe(nestedBrowseFiles);
});
it('keeps a matching folder and its entire subtree', () => {
const result = filterBrowseFiles(nestedBrowseFiles, 'bar');
expect(result).toHaveLength(1);
expect(result[0].path).toBe('bar/');
expect(result[0].children).toHaveLength(1);
expect(result[0].children[0].path).toBe('bar/bar.1.txt');
});
it('keeps the parent folders of a matching file', () => {
const result = filterBrowseFiles(nestedBrowseFiles, 'foo1');
expect(result).toHaveLength(1);
expect(result[0].path).toBe('foo/');
expect(result[0].children).toHaveLength(1);
expect(result[0].children[0].path).toBe('foo/foo1.txt');
});
it('matches case-insensitively', () => {
const result = filterBrowseFiles(nestedBrowseFiles, 'FOO1');
expect(result).toHaveLength(1);
expect(result[0].children[0].path).toBe('foo/foo1.txt');
});
it('handles the dict format returned by the API', () => {
const dictFiles = {
'foo/': {
path: 'foo/', is_empty: false, children: {
'foo1.txt': {path: 'foo/foo1.txt', size: 100},
}
},
'bar.txt': {path: 'bar.txt', size: 300},
};
const result = filterBrowseFiles(dictFiles, 'bar');
expect(result).toHaveLength(1);
expect(result[0].path).toBe('bar.txt');
});
it('returns no rows when nothing matches', () => {
expect(filterBrowseFiles(nestedBrowseFiles, 'does-not-exist')).toHaveLength(0);
});
});
it('reduces the rows to matches and their tree', async () => {
useBrowseFiles.mockReturnValue({
browseFiles: nestedBrowseFiles,
openFolders: ['foo/', 'bar/'],
setOpenFolders: jest.fn(),
fetchFiles: jest.fn(),
});
renderFileBrowser(<FileBrowser/>);
// All rows are visible before filtering.
expect(screen.getByText('foo1.txt')).toBeInTheDocument();
expect(screen.getByText('bar.1.txt')).toBeInTheDocument();
const filterInput = screen.getByPlaceholderText('Filter files...');
await act(async () => {
fireEvent.change(filterInput, {target: {value: 'bar'}});
});
// Only bar/ and its child remain.
expect(screen.getByText('bar/')).toBeInTheDocument();
expect(screen.getByText('bar.1.txt')).toBeInTheDocument();
expect(screen.queryByText('foo/')).not.toBeInTheDocument();
expect(screen.queryByText('foo1.txt')).not.toBeInTheDocument();
expect(screen.queryByText('foo2.txt')).not.toBeInTheDocument();
});
it('restores all rows when the filter is cleared', async () => {
useBrowseFiles.mockReturnValue({
browseFiles: nestedBrowseFiles,
openFolders: ['foo/', 'bar/'],
setOpenFolders: jest.fn(),
fetchFiles: jest.fn(),
});
renderFileBrowser(<FileBrowser/>);
const filterInput = screen.getByPlaceholderText('Filter files...');
await act(async () => {
fireEvent.change(filterInput, {target: {value: 'bar'}});
});
expect(screen.queryByText('foo1.txt')).not.toBeInTheDocument();
await act(async () => {
fireEvent.change(filterInput, {target: {value: ''}});
});
expect(screen.getByText('foo1.txt')).toBeInTheDocument();
expect(screen.getByText('bar.1.txt')).toBeInTheDocument();
});
it('does not filter the contents of an opened folder that matches the filter', async () => {
const mockSetOpenFolders = jest.fn();
// ebooks/ is closed, no children loaded yet.
useBrowseFiles.mockReturnValue({
browseFiles: [
{path: 'ebooks/', children: null, is_empty: false},
{path: 'other.txt', size: 100},
],
openFolders: [],
setOpenFolders: mockSetOpenFolders,
fetchFiles: jest.fn(),
});
const {rerender} = renderFileBrowser(<FileBrowser/>);
// Filter for the folder's name.
const filterInput = screen.getByPlaceholderText('Filter files...');
await act(async () => {
fireEvent.change(filterInput, {target: {value: 'ebooks'}});
});
expect(screen.queryByText('other.txt')).not.toBeInTheDocument();
// Open the matching folder.
await act(async () => {
fireEvent.click(screen.getByText('ebooks/'));
});
expect(mockSetOpenFolders).toHaveBeenCalledWith(['ebooks/']);
// Simulate the folder's children loading; their names do NOT match the filter.
useBrowseFiles.mockReturnValue({
browseFiles: [
{path: 'ebooks/', children: [
{path: 'ebooks/novel.epub', size: 100},
{path: 'ebooks/cover.jpg', size: 200},
], is_empty: false},
{path: 'other.txt', size: 100},
],
openFolders: ['ebooks/'],
setOpenFolders: mockSetOpenFolders,
fetchFiles: jest.fn(),
});
await act(async () => {
rerender(
<AllContextsWrapper>
<FileBrowser/>
</AllContextsWrapper>
);
});
// The matching folder's contents are shown unfiltered.
expect(screen.getByText('ebooks/')).toBeInTheDocument();
expect(screen.getByText('novel.epub')).toBeInTheDocument();
expect(screen.getByText('cover.jpg')).toBeInTheDocument();
// Non-matching rows outside the folder stay hidden.
expect(screen.queryByText('other.txt')).not.toBeInTheDocument();
});
it('focuses the filter input when f is pressed', async () => {
useBrowseFiles.mockReturnValue({
browseFiles: nestedBrowseFiles,
openFolders: ['foo/', 'bar/'],
setOpenFolders: jest.fn(),
fetchFiles: jest.fn(),
});
renderFileBrowser(<FileBrowser/>);
const filterInput = screen.getByPlaceholderText('Filter files...');
expect(filterInput).not.toHaveFocus();
await act(async () => {
fireEvent.keyDown(document.body, {key: 'f', code: 'KeyF'});
});
expect(filterInput).toHaveFocus();
});
it('clear button clears the filter and shows all rows again', async () => {
useBrowseFiles.mockReturnValue({
browseFiles: nestedBrowseFiles,
openFolders: ['foo/', 'bar/'],
setOpenFolders: jest.fn(),
fetchFiles: jest.fn(),
});
renderFileBrowser(<FileBrowser/>);
const clearButton = screen.getByLabelText('Clear filter');
expect(clearButton).toBeDisabled();
const filterInput = screen.getByPlaceholderText('Filter files...');
await act(async () => {
fireEvent.change(filterInput, {target: {value: 'bar'}});
});
expect(screen.queryByText('foo1.txt')).not.toBeInTheDocument();
expect(clearButton).not.toBeDisabled();
await act(async () => {
fireEvent.click(clearButton);
});
expect(filterInput).toHaveValue('');
expect(screen.getByText('foo1.txt')).toBeInTheDocument();
expect(screen.getByText('foo2.txt')).toBeInTheDocument();
expect(screen.getByText('bar.1.txt')).toBeInTheDocument();
});
it('collapse-all button is disabled when no folders are open', async () => {
useBrowseFiles.mockReturnValue({
browseFiles: nestedBrowseFiles,
openFolders: [],
setOpenFolders: jest.fn(),
fetchFiles: jest.fn(),
});
renderFileBrowser(<FileBrowser/>);
expect(screen.getByLabelText('Close all folders')).toBeDisabled();
});
it('collapse-all button closes all folders and deselects their hidden contents', async () => {
const mockSetOpenFolders = jest.fn();
useBrowseFiles.mockReturnValue({
browseFiles: nestedBrowseFiles,
openFolders: ['foo/', 'bar/'],
setOpenFolders: mockSetOpenFolders,
fetchFiles: jest.fn(),
});
renderFileBrowser(<FileBrowser/>);
// Select the foo/ folder and a file inside it.
const checkboxes = screen.getAllByRole('checkbox');
await act(async () => {
fireEvent.click(checkboxes[0]); // foo/
});
await act(async () => {
fireEvent.click(checkboxes[1]); // foo/foo1.txt
});
expect(checkboxes[0]).toBeChecked();
expect(checkboxes[1]).toBeChecked();
const collapseButton = screen.getByLabelText('Close all folders');
expect(collapseButton).not.toBeDisabled();
await act(async () => {
fireEvent.click(collapseButton);
});
// All folders are closed (clears the ?folders= query param).
expect(mockSetOpenFolders).toHaveBeenCalledWith(null);
// The folder stays selected, but its now-hidden child does not.
expect(checkboxes[0]).toBeChecked();
expect(checkboxes[1]).not.toBeChecked();
});
it('shows no results when nothing matches', async () => {
useBrowseFiles.mockReturnValue({
browseFiles: nestedBrowseFiles,
openFolders: ['foo/', 'bar/'],
setOpenFolders: jest.fn(),
fetchFiles: jest.fn(),
});
renderFileBrowser(<FileBrowser/>);
const filterInput = screen.getByPlaceholderText('Filter files...');
await act(async () => {
fireEvent.change(filterInput, {target: {value: 'does-not-exist'}});
});
expect(screen.getByText('No results')).toBeInTheDocument();
});
});
describe('Drag Selection', () => {
const mockBrowseFilesMultiple = [
{path: 'file1.txt', size: 100},

84
app/src/components/Files.js
View file

@ -40,6 +40,7 @@ import {
useSearchDate,
useSearchFiles,
useSearchCensored,
useSearchDeep,
useSearchFilter,
useSearchOrder,
useSearchView,
@ -544,6 +545,7 @@ export function SearchFilterModal(
showTags = true,
showLimit = true,
showCensored = false,
showDeep = false,
limitOptions = [12, 24, 48, 96],
},
) {
@ -553,6 +555,7 @@ export function SearchFilterModal(
const {dateRange, months} = useSearchDate();
const {activeTags, anyTag} = useSearch();
const {censored} = useSearchCensored();
const {deep} = useSearchDeep();
const {limit} = usePages(DEFAULT_SEARCH_LIMIT);
const {updateQuery} = useContext(QueryContext);
@ -570,6 +573,7 @@ export function SearchFilterModal(
const [draftMonths, setDraftMonths] = useState([]);
const [draftRange, setDraftRange] = useState(emptyRange);
const [draftCensored, setDraftCensored] = useState(false);
const [draftDeep, setDraftDeep] = useState(false);
// Seed the drafts from the URL each time the modal is opened.
React.useEffect(() => {
@ -582,6 +586,7 @@ export function SearchFilterModal(
setDraftMonths(seededMonths);
setDraftRange(seededRange);
setDraftCensored(censored || false);
setDraftDeep(deep || false);
}
}, [open]);
@ -622,6 +627,9 @@ export function SearchFilterModal(
if (showCensored && draftCensored !== censored) {
params.censored = draftCensored ? 'true' : null;
}
if (showDeep && draftDeep !== deep) {
params.deep = draftDeep ? 'true' : null;
}
if (Object.keys(params).length > 0) {
params.o = 0;
updateQuery(params);
@ -641,6 +649,7 @@ export function SearchFilterModal(
setDraftMonths([]);
setDraftRange(emptyRange);
setDraftCensored(false);
setDraftDeep(false);
}
// Sort section (operates on the draft).
@ -754,6 +763,18 @@ export function SearchFilterModal(
</Grid.Column>
</Grid.Row>}
{showDeep &&
<Grid.Row>
<Grid.Column>
<SearchFilterSection header='Search Depth'>
<Toggle label='Deep search (captions, document text; slower)'
checked={draftDeep}
onChange={setDraftDeep}
/>
</SearchFilterSection>
</Grid.Column>
</Grid.Row>}
{showDates &&
<Grid.Row>
<Grid.Column>
@ -783,6 +804,7 @@ export function SearchFilterButton(
showTags = true,
showLimit = true,
showCensored = false,
showDeep = false,
limitOptions = [12, 24, 48, 96],
size = 'medium',
content = 'Filter',
@ -794,6 +816,7 @@ export function SearchFilterButton(
const {dateRange, months} = useSearchDate();
const {activeTags, anyTag} = useSearch();
const {censored} = useSearchCensored();
const {deep} = useSearchDeep();
let count = 0;
if (fileFilterOptions && filter) {
@ -811,6 +834,9 @@ export function SearchFilterButton(
if (showCensored && censored) {
count += 1;
}
if (showDeep && deep) {
count += 1;
}
const active = count > 0;
return <>
@ -829,6 +855,7 @@ export function SearchFilterButton(
showTags={showTags}
showLimit={showLimit}
showCensored={showCensored}
showDeep={showDeep}
limitOptions={limitOptions}
/>
</>
@ -849,6 +876,7 @@ export function SearchControlBar(
fileFilterOptions = null,
showDates = false,
showCensored = false,
showDeep = false,
},
) {
const [localSearchStr, setLocalSearchStr] = useState(searchStr || '');
@ -866,17 +894,40 @@ export function SearchControlBar(
{viewButton}
<div style={{flexGrow: 1, minWidth: 0}}>{searchInput}</div>
<SearchFilterButton sorts={sorts} fileFilterOptions={fileFilterOptions} showDates={showDates}
showCensored={showCensored}/>
showCensored={showCensored} showDeep={showDeep}/>
</div>
}
// Reminds the user that the fast search only matches titles/authors/descriptions. Used by the module
// pages (Videos/Archives/Docs), which have no deep estimate, so it shows whenever a fast search is
// active; FilesSearchView has a count-aware variant.
export function DeepSearchHint({searchStr, files}) {
const {deep} = useSearchDeep();
const {updateQuery} = useContext(QueryContext);
if (!searchStr || deep || !Array.isArray(files)) {
return null;
}
return <Segment>
{files.length === 0 ? 'Not finding what you need?' : 'More results may be available in deep search.'}
<Button primary
onClick={() => updateQuery({deep: 'true', o: 0})}
style={{marginLeft: '1em'}}
>Try deep search</Button>
</Segment>
}
export function FilesSearchView({
showView = true,
emptySearch = false,
model,
deepEstimate = null,
}) {
const {searchFiles, pages} = useSearchFiles(24, emptySearch, model);
const {searchFiles, total, searchStr, pages} = useSearchFiles(24, emptySearch, model);
const {deep} = useSearchDeep();
const {updateQuery} = useContext(QueryContext);
const {body, paginator, viewButton} = FilesView(
{
@ -891,9 +942,38 @@ export function FilesSearchView({
},
);
// The default search only matches titles/authors/descriptions; offer deep search when it would find
// more. `deepEstimate` comes from the search estimates; without it, fall back to a generic hint on
// zero results.
let deepHint = null;
if (Boolean(searchStr) && !deep && searchFiles) {
const moreCount = deepEstimate !== null && deepEstimate !== undefined && total !== null
? deepEstimate - total : null;
// Enable deep search and reset pagination in a single update (like the filter modal does), so the
// user starts at page 1 of the deep results.
const deepButton = <Button primary
onClick={() => updateQuery({deep: 'true', o: 0})}
style={{marginLeft: '1em'}}
>Try deep search</Button>;
if (searchFiles.length === 0 && (moreCount === null || moreCount > 0)) {
deepHint = <Segment>
{moreCount > 0
? `No results in fast search. ${moreCount} result${moreCount === 1 ? '' : 's'} available in deep search.`
: 'Not finding what you need?'}
{deepButton}
</Segment>;
} else if (searchFiles.length > 0 && moreCount > 0) {
deepHint = <Segment>
{`${moreCount} more result${moreCount === 1 ? '' : 's'} available in deep search.`}
{deepButton}
</Segment>;
}
}
return <>
{showView && <div style={{marginBottom: '1em'}}>{viewButton}</div>}
{body}
{deepHint}
{paginator}
</>
}

26
app/src/components/Search.js
View file

@ -1,7 +1,7 @@
import React, {useState} from "react";
import {Link, Route, Routes, useLocation, useNavigate} from "react-router";
import {FilesSearchView, SearchViewButton} from "./Files";
import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchFilter} from "../hooks/customHooks";
import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchDeep, useSearchFilter} from "../hooks/customHooks";
import {ShortcutHint} from "./ShortcutHint";
import {ZimSearchView} from "./Zim";
import {MapSearchView} from "./MapSearchView";
@ -159,6 +159,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
sendGeneralReqeust(async () => await searchSuggestions(searchStr));
sendMapRequest(async () => await searchEstimateMap(searchStr));
}
// The files estimate always uses deep search server-side, so users know results are possible.
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
sendOtherRequest(async () => await searchEstimateOthers(tagNames));
@ -191,7 +192,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
React.useEffect(() => {
if (!_.isEmpty(filesData)) {
setSuggestions((prevState) => {
return {...prevState, fileGroups: filesData.fileGroups}
return {...prevState, fileGroups: filesData.fileGroups, fileGroupsDeep: filesData.fileGroupsDeep}
});
}
}, [setSuggestions, filesData]);
@ -232,6 +233,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
const {dateRange, months, setDates, clearDate} = useSearchDate();
const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter, anyTag);
const {deep} = useSearchDeep();
const {getLocationStr} = React.useContext(QueryContext);
// The results that will be displayed by <Search>.
@ -251,19 +253,25 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
let results = {};
// The dropdown always suggests the deep count so users know what is findable. The search results
// page shows the active mode's count, and offers deep search when it found less than promised.
const dropdownEstimate = newSuggestions.fileGroupsDeep ?? newSuggestions.fileGroups;
// The tab badge shows the count matching the active search mode, so it agrees with the result list.
const fileGroupsEstimate = deep ? newSuggestions.fileGroupsDeep : newSuggestions.fileGroups;
// Suggested results are ordered.
if (newSuggestions.fileGroups > 0) {
if (dropdownEstimate > 0) {
results.fileGroups = {
name: 'Files', results: [
{
title: newSuggestions.fileGroups.toString(),
title: dropdownEstimate.toString(),
type: 'files',
// Add search query onto current location.
location: getLocationStr({q: searchStr}, '/search'),
}
]
};
} else if (newSuggestions.fileGroups === 0) {
} else if (dropdownEstimate === 0) {
// Tell the user there are no files.
results.fileGroups = {name: 'Files', results: noResults};
}
@ -389,7 +397,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
setSuggestionsResults(results);
setSuggestionsSums({
fileGroups: newSuggestions.fileGroups,
fileGroups: fileGroupsEstimate,
fileGroupsDeep: newSuggestions.fileGroupsDeep,
zims: zimSum,
otherSum: otherSum,
mapPlaces: newSuggestions.mapPlaces || 0,
@ -419,7 +428,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
normalizeSuggestionsResults(suggestions);
}
}, [JSON.stringify(suggestions)]);
}, [JSON.stringify(suggestions), deep]);
// Clear timer on unmount.
React.useEffect(() => {
@ -495,7 +504,8 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
return <React.Fragment>
<TabLinks links={links} right={isFilesTab ? <SearchViewButton headlines/> : null}/>
<Routes>
<Route path='/*' element={<FilesSearchView showView={false}/>}/>
<Route path='/*' element={<FilesSearchView showView={false}
deepEstimate={suggestionsSums?.fileGroupsDeep}/>}/>
<Route path='/zim' exact element={<ZimSearchView suggestions={suggestions} loading={loading}/>}/>
<Route path='/map' exact element={<MapSearchView/>}/>
<Route path='/other' exact element={<OtherSearchView loading={loading}/>}/>

30
app/src/components/SearchFilterModal.test.js
View file

@ -146,6 +146,36 @@ describe('SearchFilterButton', () => {
expect(screen.getByText('1')).toBeInTheDocument();
});
it('counts the deep filter and applies it on close when showDeep is set', () => {
const spy = jest.fn();
renderButton({sorts: videoOrders, showDeep: true}, {}, spy);
// No deep badge initially.
expect(screen.queryByText('1')).toBeNull();
fireEvent.click(screen.getByText('Filter'));
// The Search Depth section/toggle is shown.
expect(screen.getByText('Search Depth')).toBeInTheDocument();
fireEvent.mouseUp(screen.getByTestId('toggle'));
// Draft only — nothing applied yet.
expect(spy).not.toHaveBeenCalled();
fireEvent.click(screen.getByText('Done'));
expect(spy).toHaveBeenCalledTimes(1);
expect(spy.mock.calls[0][0]).toMatchObject({deep: 'true'});
});
it('does not show the Search Depth section unless showDeep is set', () => {
renderButton({sorts: videoOrders});
fireEvent.click(screen.getByText('Filter'));
expect(screen.queryByText('Search Depth')).toBeNull();
});
it('shows a count badge when the deep filter is active in the URL', () => {
renderButton({sorts: videoOrders, showDeep: true}, {deep: 'true'});
expect(screen.getByText('1')).toBeInTheDocument();
});
it('shows the File Type section when fileFilterOptions are provided', () => {
renderButton({
sorts: videoOrders,

4
app/src/components/Videos.js
View file

@ -45,7 +45,7 @@ import {
TableCell
} from "semantic-ui-react";
import {useChannel, useSearchOrder, useSearchVideos, useVideo, useVideoStatistics} from "../hooks/customHooks";
import {FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
import {DeepSearchHint, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {Button, Card, Header, Loader, Modal, Placeholder, Popup, Segment, Statistic} from "./Theme";
@ -241,8 +241,10 @@ export function VideosPage() {
viewButton={viewButton}
sorts={videoOrders}
showCensored={true}
showDeep={true}
/>
{body}
<DeepSearchHint searchStr={searchStr} files={videos}/>
{paginator}
<TaggedDeleteConfirmModal
open={taggedFileGroups !== null}

114
app/src/components/admin/ControllerPage.js
View file

@ -11,6 +11,8 @@ import {
enableService,
getDisks,
getFstabEntries,
getHotspotDevices,
getHotspotSettings,
getSambaStatus,
getServiceLogs,
getServices,
@ -23,7 +25,9 @@ import {
startService,
stopService,
unmountDisk,
updateHotspotSettings,
} from "../../api/controller";
import QRCode from "react-qr-code";
import {toast} from "react-semantic-toasts-2";
import {RestartButton, ShutdownButton} from "./Settings";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
@ -1345,6 +1349,114 @@ function SambaSection() {
}
function HotspotSettingsForm() {
const dockerized = useDockerized();
const [devices, setDevices] = React.useState([]);
const [form, setForm] = React.useState({device: '', ssid: '', password: ''});
const [loading, setLoading] = React.useState(true);
const [saving, setSaving] = React.useState(false);
const [qrOpen, setQrOpen] = React.useState(false);
React.useEffect(() => {
const fetchSettings = async () => {
try {
const [settings, devicesResponse] = await Promise.all([getHotspotSettings(), getHotspotDevices()]);
setForm({device: settings.device, ssid: settings.ssid, password: settings.password});
setDevices(devicesResponse.devices || []);
} catch (e) {
console.error('Failed to fetch hotspot settings', e);
} finally {
setLoading(false);
}
};
fetchSettings();
}, []);
const handleSave = async () => {
setSaving(true);
try {
const settings = await updateHotspotSettings(form);
setForm(settings);
toast({type: 'success', title: 'Hotspot settings saved', time: 3000});
} catch (e) {
toast({type: 'error', title: 'Failed to save hotspot settings', description: e.message, time: 5000});
} finally {
setSaving(false);
}
};
if (loading) {
return <Segment>
<Header as='h4'>
<Icon name='wifi'/>
Hotspot Settings
</Header>
<Loader active inline size='small'/>
</Segment>;
}
// Keep the saved device selectable even if it is not currently present
// (e.g. a USB WiFi dongle is unplugged).
const deviceOptions = [...new Set([...devices, form.device].filter(Boolean))]
.map(device => ({key: device, value: device, text: device}));
// Special string which allows a mobile device to connect to a specific Wi-Fi.
// The WPA QR format requires backslash-escaping of \ ; , " in the SSID and password.
const escapeWifi = (s) => s.replace(/([\\;,"])/g, '\\$1');
const qrCodeValue = `WIFI:S:${escapeWifi(form.ssid)};T:WPA;P:${escapeWifi(form.password)};;`;
return <Segment>
<Header as='h4'>
<Icon name='wifi'/>
Hotspot Settings
</Header>
<Form>
<Form.Group widths='equal'>
<Form.Input
label='Hotspot SSID'
value={form.ssid}
disabled={dockerized || saving}
onChange={(e, {value}) => setForm({...form, ssid: value})}
/>
<Form.Input
label='Hotspot Password'
value={form.password}
disabled={dockerized || saving}
onChange={(e, {value}) => setForm({...form, password: value})}
/>
<Form.Dropdown
selection
label='Hotspot Device'
placeholder='No WiFi devices found'
options={deviceOptions}
value={form.device}
disabled={dockerized || saving}
onChange={(e, {value}) => setForm({...form, device: value})}
/>
</Form.Group>
<Button
color='violet'
disabled={dockerized}
loading={saving}
onClick={handleSave}
>Save</Button>
<Modal closeIcon
onClose={() => setQrOpen(false)}
onOpen={() => setQrOpen(true)}
open={qrOpen}
trigger={<Button icon='qrcode' color='violet'/>}
>
<Modal.Header>Scan this code to join the hotspot</Modal.Header>
<Modal.Content>
<div style={{display: 'inline-block', backgroundColor: '#ffffff', padding: '1em'}}>
<QRCode value={qrCodeValue} size={300}/>
</div>
</Modal.Content>
</Modal>
</Form>
</Segment>;
}
function AdminControlsSection() {
const dockerized = useDockerized();
@ -1359,6 +1471,8 @@ function AdminControlsSection() {
<BluetoothToggle/>
<ThrottleToggle/>
<HotspotSettingsForm/>
{!dockerized && (
<div style={{marginTop: '1em'}}>
<RestartButton/>

74
app/src/components/admin/Settings.js
View file

@ -24,7 +24,6 @@ import {
useMessageDismissal,
WROLModeMessage
} from "../Common";
import QRCode from "react-qr-code";
import {useConfigs, useDockerized} from "../../hooks/customHooks";
import {toast} from "react-semantic-toasts-2";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
@ -296,8 +295,6 @@ function WROLModeSection() {
export function SettingsPage() {
const [disabled, setDisabled] = React.useState(false);
const [editSpecialDirectories, setEditSpecialDirectories] = React.useState(false);
const [qrCodeValue, setQrCodeValue] = React.useState('');
const [qrOpen, setQrOpen] = React.useState(false);
const [ready, setReady] = React.useState(false);
const [pendingSave, setPendingSave] = React.useState(false);
const {clearAll} = useMessageDismissal();
@ -446,12 +443,6 @@ export function SettingsPage() {
setTraceModalOpen(false);
}
const handleHotspotChange = async () => {
let {hotspot_ssid, hotspot_encryption, hotspot_password} = state;
// Special string which allows a mobile device to connect to a specific Wi-Fi.
setQrCodeValue(`WIFI:S:${hotspot_ssid};T:${hotspot_encryption};P:${hotspot_password};;`);
}
React.useEffect(() => {
console.debug('settings changed, replacing state...');
setReady(settings ? true : undefined);
@ -468,10 +459,6 @@ export function SettingsPage() {
download_window_start: settings.download_window_start || '',
download_window_end: settings.download_window_end || '',
timezone: settings.timezone || '',
hotspot_device: settings.hotspot_device,
hotspot_on_startup: settings.hotspot_on_startup,
hotspot_password: settings.hotspot_password,
hotspot_ssid: settings.hotspot_ssid,
check_for_upgrades: settings.check_for_upgrades,
ignore_outdated_zims: settings.ignore_outdated_zims,
log_level: fromApiLogLevel(settings.log_level),
@ -487,10 +474,6 @@ export function SettingsPage() {
});
}, [JSON.stringify(settings)]);
React.useEffect(() => {
handleHotspotChange();
}, [state.hotspot_ssid, state.hotspot_password, state.hotspot_device]);
const handleInputChange = async (e, name, value) => {
if (e) {
e.preventDefault()
@ -507,15 +490,6 @@ export function SettingsPage() {
setState({...state, [name]: value});
}
const handleQrOpen = async (e) => {
e.preventDefault();
setQrOpen(true);
}
const qrButton = <Button icon color='violet' style={{marginBottom: '1em'}}>
<Icon name='qrcode' size='big'/>
</Button>;
const timezoneOptions = Intl.supportedValuesOf('timeZone').map(tz => (
{key: tz, value: tz, text: tz}
));
@ -619,54 +593,6 @@ export function SettingsPage() {
</ButtonGroup>
</Grid.Column>
<Grid.Column>
<Segment>
<Header as='h3'>Hotspot Settings</Header>
<Form.Input
label='Hotspot SSID'
value={state.hotspot_ssid}
disabled={disabled || state.hotspot_ssid === null}
onChange={(e, i) => setState({...state, hotspot_ssid: i.value})}
style={{marginBottom: '0.5em'}}
/>
<Form.Input
label='Hotspot Password'
disabled={disabled || state.hotspot_password === null}
value={state.hotspot_password}
onChange={(e, i) => setState({...state, hotspot_password: i.value})}
style={{marginBottom: '0.5em'}}
/>
<Form.Input
label='Hotspot Device'
disabled={disabled || state.hotspot_password === null}
value={state.hotspot_device}
onChange={(e, i) => handleInputChange(e, 'hotspot_device', i.value)}
style={{marginBottom: '0.5em'}}
/>
<Modal closeIcon
onClose={() => setQrOpen(false)}
onOpen={handleQrOpen}
open={qrOpen}
trigger={qrButton}
>
<Modal.Header>
Scan this code to join the hotspot
</Modal.Header>
<Modal.Content>
<div style={{
display: 'inline-block',
backgroundColor: '#ffffff',
padding: '1em'
}}>
<QRCode value={qrCodeValue} size={300}/>
</div>
</Modal.Content>
</Modal>
</Segment>
</Grid.Column>
</Grid.Row>
</Grid>

38
app/src/hooks/customHooks.js
View file

@ -511,6 +511,7 @@ export const useSearchArchives = (defaultLimit) => {
const activeTags = searchParams.getAll('tag');
const {view} = useSearchView();
const headline = view === 'headline';
const {deep} = useSearchDeep();
const [archives, setArchives] = useState(null);
const [totalPages, setTotalPages] = useState(0);
@ -519,7 +520,8 @@ export const useSearchArchives = (defaultLimit) => {
setArchives(null);
setTotalPages(0);
try {
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline);
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline,
deep);
setArchives(archives_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -536,7 +538,7 @@ export const useSearchArchives = (defaultLimit) => {
useEffect(() => {
localSearchArchives();
}, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), headline]);
}, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), headline, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -580,6 +582,7 @@ export const useSearchDocs = (defaultLimit) => {
const author = searchParams.get('author') || '';
const subject = searchParams.get('subject') || '';
const filter = searchParams.get('filter') || '';
const {deep} = useSearchDeep();
const [docs, setDocs] = useState(null);
const [totalPages, setTotalPages] = useState(0);
@ -589,7 +592,8 @@ export const useSearchDocs = (defaultLimit) => {
setTotalPages(0);
try {
const mimetype = docFilterToMimetype(filter);
let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject, mimetype);
let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject,
mimetype, deep);
setDocs(docs_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -600,7 +604,7 @@ export const useSearchDocs = (defaultLimit) => {
useEffect(() => {
localSearchDocs();
}, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject, filter]);
}, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject, filter, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -639,6 +643,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
const headline = view === 'headline';
const anyTag = searchParams.get('anyTag') === 'true';
const censored = searchParams.get('censored') === 'true';
const {deep} = useSearchDeep();
const [videos, setVideos] = useState(null);
const [totalPages, setTotalPages] = useState(0);
@ -649,7 +654,8 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
setLoading(true);
setTotalPages(0);
try {
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline, censored);
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline,
censored, deep);
setVideos(videos_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -662,7 +668,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
useEffect(() => {
localSearchVideos();
}, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline, censored]);
}, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline, censored, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -912,8 +918,10 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
dateRange,
} = useSearch(defaultLimit, emptySearch, model);
const {view} = useSearchView();
const {deep} = useSearchDeep();
const [searchFiles, setSearchFiles] = useState(null);
const [total, setTotal] = useState(null);
const [loading, setLoading] = useState(false);
const headline = view === 'headline';
@ -932,12 +940,14 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
setLoading(true);
console.log('localSearchFiles', fromDate, toDate, months);
try {
let [file_groups, total] = await filesSearch(
let [file_groups, total_] = await filesSearch(
pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline,
months, fromDate, toDate, anyTag);
months, fromDate, toDate, anyTag, null, null, null, deep);
setSearchFiles(file_groups);
pages.setTotal(total);
setTotal(total_);
pages.setTotal(total_);
} catch (e) {
setTotal(null);
pages.setTotal(0);
console.error(e);
toast({
@ -974,10 +984,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
JSON.stringify(months),
JSON.stringify(dateRange),
anyTag,
deep,
]);
return {
searchFiles,
total,
searchStr,
filter,
setSearchStr,
@ -1655,6 +1667,14 @@ export const useSearchCensored = () => {
return {censored, setCensored}
}
export const useSearchDeep = () => {
// Search all text including captions/document contents (?deep=true). Slower, but more thorough.
const [value, setValue] = useOneQuery('deep');
const deep = value === 'true';
const setDeep = (newValue) => setValue(newValue ? 'true' : undefined);
return {deep, setDeep}
}
export const useSearchDate = () => {
// Filters files using
// ?month=1&month=2&month=3

36
controller/api/admin.py
View file

@ -8,6 +8,9 @@ from controller.api.schemas import (
BluetoothActionResponse,
BluetoothStatusResponse,
HotspotActionResponse,
HotspotDevicesResponse,
HotspotSettingsRequest,
HotspotSettingsResponse,
HotspotStatusResponse,
RestartServicesResponse,
SambaShareAddRequest,
@ -30,13 +33,18 @@ from controller.lib.admin import (
enable_hotspot,
enable_throttle,
get_bluetooth_status_dict,
get_hotspot_device,
get_hotspot_password,
get_hotspot_ssid,
get_hotspot_status_dict,
get_throttle_status_dict,
get_timezone_status_dict,
get_wifi_devices,
reboot_system,
restart_all_services,
set_timezone,
shutdown_system,
update_hotspot_settings,
)
from controller.lib.config import is_docker_mode
from controller.lib.samba import (
@ -56,6 +64,34 @@ async def hotspot_status() -> HotspotStatusResponse:
return HotspotStatusResponse(**get_hotspot_status_dict())
@router.get("/api/hotspot/devices", response_model=HotspotDevicesResponse)
async def hotspot_devices() -> HotspotDevicesResponse:
"""List WiFi devices which could host the hotspot."""
return HotspotDevicesResponse(devices=get_wifi_devices())
@router.get("/api/hotspot/settings", response_model=HotspotSettingsResponse)
async def hotspot_settings() -> HotspotSettingsResponse:
"""Get WiFi hotspot settings."""
return HotspotSettingsResponse(
device=get_hotspot_device(),
ssid=get_hotspot_ssid(),
password=get_hotspot_password(),
)
@router.post("/api/hotspot/settings", response_model=HotspotSettingsResponse)
async def hotspot_settings_update(request: HotspotSettingsRequest) -> HotspotSettingsResponse:
"""Update WiFi hotspot settings; they are saved in controller.yaml."""
# 500 matches the other subsystem endpoints (hotspot/bluetooth/throttle) in Docker mode.
if is_docker_mode():
raise HTTPException(status_code=500, detail="Not available in Docker mode")
result = update_hotspot_settings(device=request.device, ssid=request.ssid, password=request.password)
if not result.get("success"):
raise HTTPException(status_code=400, detail=result.get("error", "Failed"))
return HotspotSettingsResponse(device=result["device"], ssid=result["ssid"], password=result["password"])
@router.post("/api/hotspot/enable", response_model=HotspotActionResponse)
async def hotspot_enable() -> HotspotActionResponse:
"""Enable WiFi hotspot."""

22
controller/api/schemas.py
View file

@ -157,6 +157,28 @@ class HotspotStatusResponse(BaseModel):
device: Optional[str] = Field(default=None, description="WiFi device name")
class HotspotDevicesResponse(BaseModel):
"""Response model for /api/hotspot/devices endpoint."""
devices: list[str] = Field(description="Available WiFi device names")
class HotspotSettingsRequest(BaseModel):
"""Request model for updating hotspot settings; only provided values are changed."""
device: Optional[str] = Field(default=None, description="WiFi device name")
ssid: Optional[str] = Field(default=None, description="Hotspot SSID")
password: Optional[str] = Field(default=None, description="Hotspot password (8 to 63 characters)")
class HotspotSettingsResponse(BaseModel):
"""Response model for /api/hotspot/settings endpoint."""
device: str = Field(description="WiFi device name")
ssid: str = Field(description="Hotspot SSID")
password: str = Field(description="Hotspot password")
class HotspotActionResponse(BaseModel):
"""Response model for hotspot enable/disable actions."""

9
controller/api/status.py
View file

@ -12,7 +12,8 @@ wrol_mode) remain in the main WROLPi API at /api/status.
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from controller.lib.admin import get_bluetooth_status, get_hotspot_status, get_throttle_status, BluetoothStatus, HotspotStatus
from controller.lib.admin import get_bluetooth_status, get_hotspot_ssid, get_hotspot_status, get_throttle_status, \
BluetoothStatus, HotspotStatus
from controller.lib.config import is_docker_mode, is_rpi, is_rpi4, is_rpi5
from controller.lib.status import (
get_cpu_status,
@ -36,16 +37,12 @@ async def get_system_info() -> dict:
Get static system info that doesn't need frequent updates.
These are collected on each request as they rarely change.
"""
from controller.lib.config import get_config
hotspot_status = get_hotspot_status()
bluetooth_status = await get_bluetooth_status()
throttle_status = get_throttle_status()
# Get hotspot SSID from config if hotspot is connected
config = get_config()
hotspot_config = config.get("hotspot", {})
hotspot_ssid = hotspot_config.get("ssid", "WROLPi") if hotspot_status == HotspotStatus.connected else None
hotspot_ssid = get_hotspot_ssid() if hotspot_status == HotspotStatus.connected else None
return {
"dockerized": is_docker_mode(),

8
controller/conftest.py
View file

@ -49,6 +49,14 @@ async def _noop_status_worker(app, base_sleep: float = 5.0):
raise
@pytest.fixture(autouse=True)
def _isolate_media_directory(tmp_path_factory, monkeypatch):
"""Point the media directory at an empty temp dir so tests never read a real
/media/wrolpi (e.g. hotspot settings from wrolpi.yaml)."""
media_dir = tmp_path_factory.mktemp("media")
monkeypatch.setenv("MEDIA_DIRECTORY", str(media_dir))
@pytest.fixture(autouse=True)
def _patch_status_worker():
"""Replace the real status worker with a no-op for every test.

148
controller/lib/admin.py
View file

@ -13,6 +13,7 @@ from pathlib import Path
import yaml
from controller.lib import config as config_lib
from controller.lib.config import is_docker_mode, get_config_value, get_media_directory
logger = logging.getLogger(__name__)
@ -45,13 +46,146 @@ class GovernorStatus(enum.Enum):
unknown = enum.auto()
def get_hotspot_device() -> str:
return get_config_value('hotspot.device', 'wlan0')
def get_hotspot_ssid() -> str:
return get_config_value('hotspot.ssid', 'WROLPi')
def get_hotspot_password() -> str:
return get_config_value('hotspot.password', 'wrolpi hotspot')
def update_hotspot_settings(device: str = None, ssid: str = None, password: str = None) -> dict:
"""
Update hotspot settings in controller.yaml. Only provided values are changed.
Returns:
dict with success status and the current settings
"""
if is_docker_mode():
return {"success": False, "error": "Not available in Docker mode"}
if device is not None and not device.strip():
return {"success": False, "error": "Hotspot device must not be empty"}
if ssid is not None and not ssid.strip():
return {"success": False, "error": "Hotspot SSID must not be empty"}
# nmcli requires a WPA password of 8 to 63 characters.
if password is not None and not 8 <= len(password) <= 63:
return {"success": False, "error": "Hotspot password must be 8 to 63 characters"}
changed = False
if device:
config_lib.update_config('hotspot.device', device)
changed = True
if ssid:
config_lib.update_config('hotspot.ssid', ssid)
changed = True
if password:
config_lib.update_config('hotspot.password', password)
changed = True
if changed:
try:
config_lib.save_config()
except RuntimeError as e:
return {"success": False, "error": str(e)}
logger.info("Hotspot settings updated (device=%s, ssid=%s)", get_hotspot_device(), get_hotspot_ssid())
return {
"success": True,
"device": get_hotspot_device(),
"ssid": get_hotspot_ssid(),
"password": get_hotspot_password(),
}
def migrate_hotspot_settings_from_wrolpi_config():
"""
One-time copy of hotspot settings from wrolpi.yaml into controller.yaml.
Hotspot settings used to be saved in wrolpi.yaml by the Settings page; the
Controller now owns them in controller.yaml. Copy any values the user set
in wrolpi.yaml unless controller.yaml already sets them explicitly.
Called on Controller startup after the primary drive is mounted.
"""
if not config_lib.CONFIG_PATH_ON_DRIVE.exists():
return
try:
with open(config_lib.CONFIG_PATH_ON_DRIVE) as f:
controller_config = yaml.safe_load(f) or {}
except (IOError, yaml.YAMLError) as e:
logger.warning("Failed to read controller.yaml for hotspot migration: %s", e)
return
wrolpi_config_path = get_media_directory() / 'config' / 'wrolpi.yaml'
if not wrolpi_config_path.exists():
return
try:
with open(wrolpi_config_path) as f:
wrolpi_config = yaml.safe_load(f) or {}
except (IOError, yaml.YAMLError) as e:
logger.warning("Failed to read wrolpi.yaml for hotspot migration: %s", e)
return
explicit = controller_config.get('hotspot') or {}
key_map = {'device': 'hotspot_device', 'ssid': 'hotspot_ssid', 'password': 'hotspot_password'}
changed = False
for controller_key, wrolpi_key in key_map.items():
value = wrolpi_config.get(wrolpi_key)
if value and controller_key not in explicit and value != get_config_value(f'hotspot.{controller_key}'):
config_lib.update_config(f'hotspot.{controller_key}', value)
changed = True
if changed:
try:
config_lib.save_config()
logger.info("Migrated hotspot settings from wrolpi.yaml to controller.yaml")
except RuntimeError as e:
logger.warning("Could not save migrated hotspot settings: %s", e)
def get_wifi_devices() -> list[str]:
"""
List WiFi device names (e.g. wlan0, wlp2s0) using nmcli.
Returns an empty list in Docker mode or when nmcli is unavailable.
"""
if is_docker_mode():
return []
try:
result = subprocess.run(
["nmcli", "-t", "-f", "DEVICE,TYPE", "device"],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode != 0:
return []
devices = []
for line in result.stdout.strip().splitlines():
parts = line.split(":")
# Exact "wifi" match excludes wifi-p2p devices.
if len(parts) >= 2 and parts[1] == "wifi":
devices.append(parts[0])
return devices
except (FileNotFoundError, subprocess.TimeoutExpired, subprocess.SubprocessError):
return []
def get_current_ssid(interface: str = None) -> str | None:
"""
Returns the name of the SSID that is currently connected.
Returns None if Hotspot is active, or no Wi-Fi network is being used.
"""
if interface is None:
interface = get_config_value('hotspot.device', 'wlan0')
interface = get_hotspot_device()
try:
result = subprocess.run(
['iwgetid', interface, '--raw'],
@ -81,7 +215,7 @@ def get_hotspot_status() -> HotspotStatus:
if is_docker_mode():
return HotspotStatus.unknown
device = get_config_value('hotspot.device', 'wlan0')
device = get_hotspot_device()
# Check if device is connected to a Wi-Fi network (not hotspot)
if get_current_ssid(device):
@ -128,7 +262,7 @@ def get_hotspot_status_dict() -> dict:
ssid: Optional[str] - Hotspot SSID when enabled
device: Optional[str] - WiFi device name
"""
device = get_config_value('hotspot.device', 'wlan0')
device = get_hotspot_device()
status = get_hotspot_status()
# Map enum status to enabled/available flags
@ -147,7 +281,7 @@ def get_hotspot_status_dict() -> dict:
"enabled": enabled,
"available": available,
"reason": reason,
"ssid": get_config_value('hotspot.ssid', 'WROLPi') if enabled else None,
"ssid": get_hotspot_ssid() if enabled else None,
"device": device,
}
@ -162,9 +296,9 @@ def enable_hotspot() -> dict:
if is_docker_mode():
return {"success": False, "error": "Not available in Docker mode"}
device = get_config_value('hotspot.device', 'wlan0')
ssid = get_config_value('hotspot.ssid', 'WROLPi')
password = get_config_value('hotspot.password', 'wrolpi hotspot')
device = get_hotspot_device()
ssid = get_hotspot_ssid()
password = get_hotspot_password()
try:
# First ensure radio is on

8
controller/main.py
View file

@ -87,9 +87,12 @@ async def lifespan(app: FastAPI):
logger.info("Using default configuration (no controller.yaml found)")
# Apply timezone from wrolpi.yaml to the system.
from controller.lib.admin import apply_timezone_from_config
from controller.lib.admin import apply_timezone_from_config, migrate_hotspot_settings_from_wrolpi_config
apply_timezone_from_config()
# Copy legacy hotspot settings from wrolpi.yaml into controller.yaml.
migrate_hotspot_settings_from_wrolpi_config()
# Apply Samba config if enabled.
from controller.lib.samba import apply_samba_from_config
apply_samba_from_config()
@ -108,8 +111,9 @@ async def lifespan(app: FastAPI):
except RuntimeError as e:
logger.warning("Failed to create controller.yaml: %s", e)
from controller.lib.admin import apply_timezone_from_config
from controller.lib.admin import apply_timezone_from_config, migrate_hotspot_settings_from_wrolpi_config
apply_timezone_from_config()
migrate_hotspot_settings_from_wrolpi_config()
from controller.lib.samba import apply_samba_from_config
apply_samba_from_config()

60
controller/templates/index.html
View file

@ -959,6 +959,23 @@
<button id="hotspot-btn" class="btn" onclick="toggleHotspot()" disabled>Loading...</button>
<button id="bluetooth-btn" class="btn" onclick="toggleBluetooth()" disabled>Loading...</button>
</div>
<details style="margin-bottom: 1rem;">
<summary style="cursor: pointer;">Hotspot Settings</summary>
<div class="modal-form" style="max-width: 24rem; margin-top: 0.5rem;">
<label>SSID
<input type="text" id="hotspot-ssid">
</label>
<label>Password
<input type="text" id="hotspot-password">
</label>
<label>Device
<select id="hotspot-device"
style="width: 100%; padding: 0.5rem; margin-bottom: 1rem; background: #1a1a2e; border: 1px solid #333; border-radius: 4px; color: #fff; font-size: 1rem;">
</select>
</label>
<button class="btn btn-primary" onclick="saveHotspotSettings()">Save Hotspot Settings</button>
</div>
</details>
{% endif %}
<div class="action-buttons">
<button class="btn btn-warning" onclick="restartServices()">Restart All Services</button>
@ -1372,6 +1389,48 @@
await updateHotspotButton();
}
async function loadHotspotSettings() {
const ssidInput = document.getElementById('hotspot-ssid');
if (!ssidInput) return;
try {
const [settings, devices] = await Promise.all([
apiCall('api/hotspot/settings'),
apiCall('api/hotspot/devices'),
]);
ssidInput.value = settings.ssid;
document.getElementById('hotspot-password').value = settings.password;
const select = document.getElementById('hotspot-device');
select.innerHTML = '';
// Keep the saved device selectable even if it is not currently present.
const names = [...new Set([...(devices.devices || []), settings.device])].filter(Boolean);
for (const name of names) {
const option = document.createElement('option');
option.value = name;
option.textContent = name;
option.selected = name === settings.device;
select.appendChild(option);
}
} catch (e) {
console.error('Failed to load hotspot settings', e);
}
}
async function saveHotspotSettings() {
const btn = event.target;
btn.disabled = true;
try {
await apiCall('api/hotspot/settings', 'POST', {
ssid: document.getElementById('hotspot-ssid').value,
password: document.getElementById('hotspot-password').value,
device: document.getElementById('hotspot-device').value,
});
await updateHotspotButton();
} catch (e) {
alert(`Failed to save hotspot settings: ${e.message}`);
}
btn.disabled = false;
}
// --- Bluetooth ---
let bluetoothEnabled = null;
@ -2759,6 +2818,7 @@
// Fetch hardware button status immediately
updateHotspotButton();
updateBluetoothButton();
loadHotspotSettings();
// Initial update after a short delay (to not overlap with initial load)
setTimeout(() => {

168
controller/test/test_admin.py
View file

@ -5,7 +5,9 @@ Unit tests for controller.lib.admin module.
from unittest import mock
import pytest
import yaml
from controller.lib.config import reload_config_from_drive
from controller.lib.admin import (
apply_timezone_from_config,
disable_bluetooth,
@ -19,10 +21,16 @@ from controller.lib.admin import (
get_hotspot_status_dict,
get_throttle_status,
get_timezone_status_dict,
get_hotspot_device,
get_hotspot_password,
get_hotspot_ssid,
get_wifi_devices,
migrate_hotspot_settings_from_wrolpi_config,
reboot_system,
restart_all_services,
set_timezone,
shutdown_system,
update_hotspot_settings,
BluetoothStatus,
HotspotStatus,
GovernorStatus,
@ -295,6 +303,166 @@ class TestGetHotspotStatusDict:
assert result["enabled"] is True
class TestHotspotSettingsMigration:
"""Legacy hotspot settings in wrolpi.yaml are copied into controller.yaml on startup."""
@staticmethod
def _write_wrolpi_yaml(test_config_directory, **values):
lines = ''.join(f'{key}: {value}\n' for key, value in values.items())
(test_config_directory / 'wrolpi.yaml').write_text(lines)
def test_migrates_wrolpi_yaml_values(self, test_directory, test_config_directory, mock_config_path,
reset_runtime_config):
"""wrolpi.yaml hotspot settings should be copied to controller.yaml."""
mock_config_path.write_text("{}\n")
self._write_wrolpi_yaml(test_config_directory, hotspot_device='wlp2s0', hotspot_ssid='RafaelPi',
hotspot_password='password123')
with mock.patch("controller.lib.admin.get_media_directory", return_value=test_directory):
migrate_hotspot_settings_from_wrolpi_config()
assert get_hotspot_device() == 'wlp2s0'
assert get_hotspot_ssid() == 'RafaelPi'
assert get_hotspot_password() == 'password123'
# The migrated values are persisted in controller.yaml.
saved = yaml.safe_load(mock_config_path.read_text())
assert saved['hotspot'] == {'device': 'wlp2s0', 'ssid': 'RafaelPi', 'password': 'password123'}
def test_does_not_overwrite_explicit_controller_values(self, test_directory, test_config_directory,
mock_config_path, reset_runtime_config):
"""Explicit controller.yaml values win over legacy wrolpi.yaml values."""
mock_config_path.write_text("hotspot:\n device: wlan1\n")
reload_config_from_drive()
self._write_wrolpi_yaml(test_config_directory, hotspot_device='wlp2s0', hotspot_ssid='RafaelPi')
with mock.patch("controller.lib.admin.get_media_directory", return_value=test_directory):
migrate_hotspot_settings_from_wrolpi_config()
# The explicit device is kept, but the SSID is still migrated.
assert get_hotspot_device() == 'wlan1'
assert get_hotspot_ssid() == 'RafaelPi'
def test_noop_without_wrolpi_yaml(self, test_directory, test_config_directory, mock_config_path,
reset_runtime_config):
"""Nothing happens when wrolpi.yaml does not exist."""
mock_config_path.write_text("{}\n")
with mock.patch("controller.lib.admin.get_media_directory", return_value=test_directory):
migrate_hotspot_settings_from_wrolpi_config()
assert get_hotspot_device() == 'wlan0'
assert yaml.safe_load(mock_config_path.read_text()) == {}
def test_noop_when_drive_not_mounted(self, test_directory, test_config_directory, mock_config_path,
reset_runtime_config):
"""Nothing happens when controller.yaml does not exist (drive not mounted)."""
self._write_wrolpi_yaml(test_config_directory, hotspot_device='wlp2s0')
with mock.patch("controller.lib.admin.get_media_directory", return_value=test_directory):
migrate_hotspot_settings_from_wrolpi_config()
assert get_hotspot_device() == 'wlan0'
class TestUpdateHotspotSettings:
"""Tests for update_hotspot_settings function."""
def test_updates_and_saves(self, mock_config_path, reset_runtime_config):
"""New settings should be applied to the runtime config and saved to controller.yaml."""
with mock.patch("controller.lib.admin.is_docker_mode", return_value=False):
result = update_hotspot_settings(device='wlp2s0', ssid='RafaelPi', password='password123')
assert result["success"] is True
assert get_hotspot_device() == 'wlp2s0'
saved = yaml.safe_load(mock_config_path.read_text())
assert saved['hotspot'] == {'device': 'wlp2s0', 'ssid': 'RafaelPi', 'password': 'password123'}
def test_partial_update(self, mock_config_path, reset_runtime_config):
"""Only provided values are changed."""
with mock.patch("controller.lib.admin.is_docker_mode", return_value=False):
result = update_hotspot_settings(device='wlp2s0')
assert result["success"] is True
assert result["device"] == 'wlp2s0'
assert result["ssid"] == 'WROLPi'
assert result["password"] == 'wrolpi hotspot'
def test_rejects_short_password(self, mock_config_path, reset_runtime_config):
"""nmcli requires a WPA password of 8 to 63 characters."""
with mock.patch("controller.lib.admin.is_docker_mode", return_value=False):
result = update_hotspot_settings(password='short')
assert result["success"] is False
assert "8" in result["error"]
assert get_hotspot_password() == 'wrolpi hotspot'
def test_rejects_in_docker_mode(self, reset_runtime_config):
"""Hotspot settings cannot be changed in Docker mode."""
with mock.patch("controller.lib.admin.is_docker_mode", return_value=True):
result = update_hotspot_settings(device='wlp2s0')
assert result["success"] is False
assert "Docker" in result["error"]
def test_noop_does_not_save(self, tmp_path, reset_runtime_config):
"""An empty update succeeds without writing controller.yaml."""
missing_path = tmp_path / 'missing' / 'controller.yaml'
with mock.patch("controller.lib.admin.is_docker_mode", return_value=False):
# save_config() would raise RuntimeError here; success proves it was skipped.
with mock.patch("controller.lib.config.CONFIG_PATH_ON_DRIVE", missing_path):
result = update_hotspot_settings()
assert result["success"] is True
assert not missing_path.exists()
def test_error_when_drive_not_mounted(self, tmp_path, reset_runtime_config):
"""Saving fails cleanly when the primary drive is not mounted."""
missing_path = tmp_path / 'missing' / 'controller.yaml'
with mock.patch("controller.lib.admin.is_docker_mode", return_value=False):
with mock.patch("controller.lib.config.CONFIG_PATH_ON_DRIVE", missing_path):
result = update_hotspot_settings(device='wlp2s0')
assert result["success"] is False
assert "not mounted" in result["error"]
class TestGetWifiDevices:
"""Tests for get_wifi_devices function."""
# Real terse nmcli output from an x86 PC: `nmcli -t -f DEVICE,TYPE device`
NMCLI_TERSE_DEVICES = (
"eno1:ethernet\n"
"wlp2s0:wifi\n"
"p2p-dev-wlp2s0:wifi-p2p\n"
"lo:loopback\n"
)
def test_returns_wifi_devices_only(self):
"""Should return only wifi-type devices, excluding wifi-p2p and ethernet."""
mock_result = mock.Mock(returncode=0, stdout=self.NMCLI_TERSE_DEVICES)
with mock.patch("controller.lib.admin.is_docker_mode", return_value=False):
with mock.patch("subprocess.run", return_value=mock_result):
assert get_wifi_devices() == ["wlp2s0"]
def test_returns_empty_in_docker_mode(self):
"""Should return an empty list in Docker mode."""
with mock.patch("controller.lib.admin.is_docker_mode", return_value=True):
assert get_wifi_devices() == []
def test_returns_empty_when_nmcli_missing(self):
"""Should return an empty list when nmcli is not installed."""
with mock.patch("controller.lib.admin.is_docker_mode", return_value=False):
with mock.patch("subprocess.run", side_effect=FileNotFoundError()):
assert get_wifi_devices() == []
def test_returns_empty_when_nmcli_fails(self):
"""Should return an empty list when nmcli returns an error."""
mock_result = mock.Mock(returncode=8, stdout="")
with mock.patch("controller.lib.admin.is_docker_mode", return_value=False):
with mock.patch("subprocess.run", return_value=mock_result):
assert get_wifi_devices() == []
class TestDockerModeErrors:
"""Admin functions should return an error result when running in Docker mode."""

48
controller/test/test_admin_api.py
View file

@ -34,6 +34,54 @@ class TestStatusEndpoints:
assert data["available"] is False
class TestHotspotDevicesEndpoint:
"""GET /api/hotspot/devices lists WiFi interfaces for the Settings dropdown."""
def test_devices_endpoint_shape(self, test_client):
"""Should return the WiFi devices found by nmcli."""
from unittest import mock
with mock.patch("controller.api.admin.get_wifi_devices", return_value=["wlan0", "wlp2s0"]):
response = test_client.get("/api/hotspot/devices")
assert response.status_code == 200
assert response.json() == {"devices": ["wlan0", "wlp2s0"]}
def test_devices_empty_in_docker(self, test_client_docker_mode):
"""Should return an empty list in Docker mode."""
response = test_client_docker_mode.get("/api/hotspot/devices")
assert response.status_code == 200
assert response.json() == {"devices": []}
class TestHotspotSettingsEndpoint:
"""GET/POST /api/hotspot/settings manage hotspot settings in controller.yaml."""
def test_get_settings(self, test_client):
"""Should return the current hotspot settings."""
response = test_client.get("/api/hotspot/settings")
assert response.status_code == 200
assert response.json() == {"device": "wlan0", "ssid": "WROLPi", "password": "wrolpi hotspot"}
def test_post_settings(self, test_client, mock_config_path):
"""Should update settings and persist them to controller.yaml."""
response = test_client.post(
"/api/hotspot/settings",
json={"device": "wlp2s0", "ssid": "RafaelPi", "password": "password123"},
)
assert response.status_code == 200
assert response.json() == {"device": "wlp2s0", "ssid": "RafaelPi", "password": "password123"}
assert "wlp2s0" in mock_config_path.read_text()
def test_post_rejects_short_password(self, test_client, mock_config_path):
"""Should reject a password shorter than 8 characters."""
response = test_client.post("/api/hotspot/settings", json={"password": "short"})
assert response.status_code == 400
def test_post_rejected_in_docker(self, test_client_docker_mode):
"""Should reject settings changes in Docker mode."""
response = test_client_docker_mode.post("/api/hotspot/settings", json={"device": "wlp2s0"})
assert response.status_code == 500
class TestDockerModeRejectsAdminActions:
"""Admin action endpoints should reject requests when running in Docker."""

4
debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh
View file

@ -59,7 +59,7 @@ progress "Preparing WROLPi..."
# --- Detect deployment mode ---------------------------------------------
# Three mutually-exclusive shapes:
#
# direct Live USB flashed with dd/Etcher: the ISO partition is a
# direct Live USB flashed with dd/Raspberry Pi Imager: the ISO partition is a
# real partition on a real disk. Create + manage a
# persistence partition on that disk.
#
@ -326,7 +326,7 @@ WROLPi can't safely write a persistence partition in that case, so
everything lives in RAM and will be lost when you reboot or power off.
To get a persistent install, flash the ISO directly to a USB drive
with dd or Etcher and boot from that drive instead.
with dd or Raspberry Pi Imager and boot from that drive instead.
EOF
else
cat > "$SUMMARY_FILE" <<EOF

2
docker/help/wrolpi-help

@ -1 +1 @@
Subproject commit 002e747fba33057a176451ec7f5d5f347e0f1c16
Subproject commit 8bd80a152a08eed265d248f0acb4e4881f4035ce

2
modules/archive/api.py
View file

@ -61,7 +61,7 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
offset = body.offset or 0
file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names,
body.headline)
body.headline, body.deep)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)

8
modules/archive/lib.py
View file

@ -1400,7 +1400,7 @@ NO_NULL_ORDERS = {
def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str],
headline: bool = False) \
headline: bool = False, deep: bool = False) \
-> Tuple[List[dict], int]:
# Always filter FileGroups to Archives.
wheres = []
@ -1412,9 +1412,11 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
select_columns = ''
if search_str:
# A search_str was provided by the user, modify the query to filter by it.
select_columns = 'ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)) AS rank,' \
# `textsearch` includes d_text (page contents); `textsearch_abc` is much smaller and faster.
ts_col = 'textsearch' if deep else 'textsearch_abc'
select_columns = f'ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)) AS rank,' \
' COUNT(*) OVER() AS total'
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
params['search_str'] = search_str
group_by = f'{group_by}, rank'

1
modules/archive/schema.py
View file

@ -24,6 +24,7 @@ class ArchiveSearchRequest:
order_by: Optional[str] = None
tag_names: List[str] = field(default_factory=list)
headline: bool = False
deep: bool = False # When True, search all text including page contents (d_text)
@dataclass

28
modules/archive/test/test_api.py
View file

@ -50,36 +50,43 @@ async def test_archives_search(test_session, archive_directory, archive_factory,
archive_factory('example.org') # has no contents
test_session.commit()
# Contents are d_text, so a deep search is required to match them.
# 1 and 2 contain "foo".
data = {'search_str': 'foo'}
data = {'search_str': 'foo', 'deep': True}
await check_results(async_client, data, [2, 1])
# The default (fast) search does not match contents.
data = {'search_str': 'foo'}
await check_results(async_client, data, [])
# 2 and 3 contain "baz".
data = {'search_str': 'baz'}
data = {'search_str': 'baz', 'deep': True}
await check_results(async_client, data, [3, 2])
# 1 contains "bar".
data = {'search_str': 'bar'}
data = {'search_str': 'bar', 'deep': True}
await check_results(async_client, data, [1, ])
# No archives contain "huzzah"
data = {'search_str': 'huzzah'}
data = {'search_str': 'huzzah', 'deep': True}
await check_results(async_client, data, [])
# Only 3 contains "baz" and is in domain "example.org"
data = {'search_str': 'baz', 'domain': 'example.org'}
data = {'search_str': 'baz', 'domain': 'example.org', 'deep': True}
await check_results(async_client, data, [3, ])
# 1's title contains "my", this is ignored by Postgres.
data = {'search_str': 'my'}
data = {'search_str': 'my', 'deep': True}
await check_results(async_client, data, [])
# 3's title contains "third".
# 3's title contains "third", found by the default (fast) search and the deep search.
data = {'search_str': 'third'}
await check_results(async_client, data, [3, ])
data = {'search_str': 'third', 'deep': True}
await check_results(async_client, data, [3, ])
# All contents contain "qux", but they contain different amounts. They are ordered by the amount.
data = {'search_str': 'qux'}
data = {'search_str': 'qux', 'deep': True}
await check_results(async_client, data, [3, 2, 1])
data = {'search_str': 'qux', 'order_by': 'bad order_by'}
@ -116,7 +123,8 @@ async def test_archives_search_headline(test_session, archive_directory, archive
archive_factory('example.org') # has no contents
test_session.commit()
content = dict(search_str='foo')
# "foo" is only in the contents (d_text), so a deep search is required to match it.
content = dict(search_str='foo', deep=True)
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
@ -124,7 +132,7 @@ async def test_archives_search_headline(test_session, archive_directory, archive
assert response.json['file_groups'][0]['d_headline'] is None
assert response.json['file_groups'][1]['d_headline'] is None
content = dict(search_str='foo', headline=True)
content = dict(search_str='foo', headline=True, deep=True)
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK

3
modules/archive/test/test_lib.py
View file

@ -571,7 +571,8 @@ async def test_refresh_archives(test_session, test_directory, async_client, make
assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file()
assert test_session.query(Archive).count() == 2
content = json.dumps({'search_str': 'text', 'model': 'archive'})
# "text" is only in the readability contents (d_text), so a deep search is required.
content = json.dumps({'search_str': 'text', 'model': 'archive', 'deep': True})
request, response = await async_client.post('/api/files/search', content=content)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], 'No files matched "text"'

1
modules/docs/api.py
View file

@ -56,6 +56,7 @@ async def search_docs(_: Request, body: schema.DocSearchRequest):
offset=offset,
order_by=body.order_by,
tag_names=body.tag_names,
deep=body.deep,
)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)

9
modules/docs/lib.py
View file

@ -347,15 +347,18 @@ def _get_doc(session, file_group_id: int):
def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
limit=20, offset=0, order_by='published_datetime', tag_names=None):
limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False):
from .models import Doc
from wrolpi.files.models import FileGroup
# `textsearch` includes d_text (document contents); `textsearch_abc` is much smaller and faster.
ts_col = FileGroup.textsearch if deep else FileGroup.textsearch_abc
with get_db_session() as session:
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
if search_str:
query = query.filter(FileGroup.textsearch.op('@@')(func.websearch_to_tsquery(search_str)))
query = query.filter(ts_col.op('@@')(func.websearch_to_tsquery(search_str)))
if author:
query = query.filter(FileGroup.b_text.ilike(f'%{author}%'))
@ -382,7 +385,7 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
# Ordering.
if order_by == 'rank' and search_str:
query = query.order_by(
desc(func.ts_rank(FileGroup.textsearch, func.websearch_to_tsquery(search_str))),
desc(func.ts_rank(ts_col, func.websearch_to_tsquery(search_str))),
desc(FileGroup.id),
)
elif order_by == 'published_datetime':

1
modules/docs/schema.py
View file

@ -13,6 +13,7 @@ class DocSearchRequest:
order_by: Optional[str] = 'published_datetime'
limit: int = 20
offset: int = 0
deep: bool = False # When True, search all text including document contents (d_text)
@dataclasses.dataclass

6
modules/docs/test/test_lib.py
View file

@ -289,7 +289,11 @@ async def test_search_docs_attaches_section_hint_epub(test_session, test_directo
(2, 'Chapter 2', 'unrelated content about pottery'),
])
# "mullen" is only in the document body (d_text), so a deep search is required to match it.
results, total = _search_docs(search_str='mullen', mimetype='application/epub')
assert total == 0, 'Document contents are not searched unless deep=True'
results, total = _search_docs(search_str='mullen', mimetype='application/epub', deep=True)
assert total == 1
assert results[0]['id'] == fg.id
hint = results[0]['section_hint']
@ -311,7 +315,7 @@ async def test_search_docs_attaches_section_hint_pdf(test_session, test_director
(3, 'Page 3', 'here is where mullen shows up'),
])
results, total = _search_docs(search_str='mullen', mimetype='application/pdf')
results, total = _search_docs(search_str='mullen', mimetype='application/pdf', deep=True)
assert total == 1
assert results[0]['id'] == fg.id
hint = results[0]['section_hint']

65
modules/videos/channel/test/test_models.py
View file

@ -110,6 +110,71 @@ async def test_channel_download_relationships(test_session, download_channel):
assert test_session.query(Download).count() == 0
@pytest.mark.asyncio
async def test_channel_statistics_with_tagged_videos(async_client, test_session, channel_factory, video_factory,
tag_factory):
"""Tagging a Video must not inflate its Channel's statistics.
A Video with multiple Tags was counted once per Tag because of the tag_file join."""
channel = channel_factory()
video1 = video_factory(channel_id=channel.id)
video2 = video_factory(channel_id=channel.id)
video3 = video_factory(channel_id=channel.id)
test_session.commit()
expected_size = video1.file_group.size + video2.file_group.size + video3.file_group.size
# Statistics are correct before any Videos are tagged.
statistics = channel.get_statistics()
assert statistics['video_count'] == 3
assert statistics['size'] == expected_size
assert statistics['video_tags'] == 0
# video1 has two Tags, video2 has one Tag, video3 has none.
tag1, tag2 = await tag_factory(), await tag_factory()
video1.add_tag(test_session, tag1.name)
video1.add_tag(test_session, tag2.name)
video2.add_tag(test_session, tag1.name)
test_session.commit()
# Tagging Videos must not change the number of Videos in the Channel.
statistics = channel.get_statistics()
assert statistics['video_count'] == 3
assert statistics['size'] == expected_size
assert statistics['video_tags'] == 2
def test_channel_directory_video_counts(test_session, channel_factory, video_factory):
"""A Channel can count the video FileGroups in its directory, and which Channel they are assigned to.
These counts are logged after a Channel refresh to help debug Videos missing from their Channel."""
channel1 = channel_factory()
channel2 = channel_factory()
video_factory(channel_id=channel1.id)
video_factory(channel_id=channel1.id)
# These Videos' files are in channel1's directory, but they are not assigned to channel1.
other_channel_video = video_factory(with_video_file=channel1.directory / 'other channel.mp4')
unassigned_video = video_factory(with_video_file=channel1.directory / 'unassigned.mp4')
other_channel_video.channel_id = channel2.id
unassigned_video.channel_id = None
test_session.commit()
assert channel1.get_directory_video_counts() == dict(
video_file_groups=4,
assigned_to_channel=2,
assigned_to_other_channels=1,
unassigned=1,
)
# channel2's directory contains no files.
assert channel2.get_directory_video_counts() == dict(
video_file_groups=0,
assigned_to_channel=0,
assigned_to_other_channels=0,
unassigned=0,
)
def test_channel_info_json(test_session, test_directory):
from wrolpi.collections import Collection

3
modules/videos/conftest.py
View file

@ -336,6 +336,7 @@ def assert_video_search(async_client):
order_by: str = None,
channel_id: int = None,
censored: bool = None,
deep: bool = None,
):
content = dict()
if search_str is not None:
@ -352,6 +353,8 @@ def assert_video_search(async_client):
content['channel_id'] = channel_id
if censored is not None:
content['censored'] = censored
if deep is not None:
content['deep'] = deep
request, response = await async_client.post('/api/videos/search', content=json.dumps(content))

42
modules/videos/models.py
View file

@ -953,20 +953,42 @@ class Channel(ModelHelper, Base):
"""Get statistics about this channel."""
with get_db_curs() as curs:
stmt = '''
SELECT SUM(size) AS "size",
MAX(size) AS "largest_video",
COUNT(video.id) AS "video_count",
SUM(fg.length) AS "length",
-- Videos may use multiple tags.
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
SELECT SUM(size) AS "size",
MAX(size) AS "largest_video",
COUNT(video.id) AS "video_count",
SUM(fg.length) AS "length",
-- Count Videos with at least one Tag; a Video may use multiple Tags.
(SELECT COUNT(DISTINCT tf.file_group_id)
FROM tag_file tf
INNER JOIN video v ON v.file_group_id = tf.file_group_id
WHERE v.channel_id = %(id)s) AS "video_tags"
FROM video
LEFT JOIN file_group fg on fg.id = video.file_group_id
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
WHERE channel_id = %(id)s \
'''
curs.execute(stmt, dict(id=self.id))
return dict(curs.fetchone())
def get_directory_video_counts(self) -> dict:
"""Count the video/audio FileGroups within my directory, and which Channel they are assigned to.
Logged after a refresh to help debug Videos missing from their Channel."""
with get_db_curs() as curs:
stmt = '''
SELECT COUNT(fg.id) AS "video_file_groups",
COUNT(fg.id) FILTER ( WHERE v.channel_id = %(id)s ) AS "assigned_to_channel",
COUNT(fg.id) FILTER ( WHERE v.channel_id IS NOT NULL
AND v.channel_id != %(id)s ) AS "assigned_to_other_channels",
COUNT(fg.id) FILTER ( WHERE v.id IS NULL
OR v.channel_id IS NULL ) AS "unassigned"
FROM file_group fg
LEFT JOIN video v ON v.file_group_id = fg.id
WHERE (fg.directory = %(directory)s OR fg.directory LIKE %(directory)s || '/%%')
AND (fg.mimetype LIKE 'video/%%' OR fg.mimetype LIKE 'audio/%%') \
'''
curs.execute(stmt, dict(id=self.id, directory=str(self.directory)))
return dict(curs.fetchone())
def set_tag(self, tag_id_or_name: int | str | None) -> Tag | None:
"""Change the Tag relationship of this Channel. Will clear the Tag if provided with None."""
session = Session.object_session(self)
@ -1016,6 +1038,12 @@ class Channel(ModelHelper, Base):
with get_db_session(commit=True) as session_:
channel_ = cls.find_by_id(session_, id_)
channel_.refreshed = True
counts = channel_.get_directory_video_counts()
logger.info(f'Refreshed {channel_}:'
f' {counts["video_file_groups"]} video FileGroups in directory,'
f' {counts["assigned_to_channel"]} assigned to this Channel,'
f' {counts["assigned_to_other_channels"]} assigned to other Channels,'
f' {counts["unassigned"]} unassigned')
background_task(_())
return job_id

1
modules/videos/schema.py
View file

@ -178,6 +178,7 @@ class VideoSearchRequest:
tag_names: List[str] = field(default_factory=list)
headline: bool = False
censored: bool = False
deep: bool = False # When True, search all text including captions (d_text)
@dataclass

21
modules/videos/test/test_api.py
View file

@ -195,31 +195,36 @@ async def test_search_videos_file(test_session, test_directory, video_with_searc
video_with_search_factory(title=title, d_text=caption)
test_session.commit()
# Captions are d_text, so a deep search is required to match them.
# Repeated runs should return the same result
for _ in range(2):
# Only videos with a b are returned, ordered by the amount of b's
await assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
await assert_video_search(search_str='b', deep=True, assert_ids=[1, 2, 3, 4])
# The default (fast) search does not match captions.
await assert_video_search(search_str='b', assert_ids=[])
# Only two captions have e
await assert_video_search(search_str='e', assert_ids=[4, 1])
await assert_video_search(search_str='e', deep=True, assert_ids=[4, 1])
# Only 2 contains 2
await assert_video_search(search_str='2', assert_ids=[2, ])
await assert_video_search(search_str='2', deep=True, assert_ids=[2, ])
# Only two captions have d
await assert_video_search(search_str='d', assert_ids=[1, 2])
await assert_video_search(search_str='d', deep=True, assert_ids=[1, 2])
# 5 can be gotten by it's title
# 5 can be gotten by it's title, in either mode.
await assert_video_search(search_str='5', assert_ids=[5, ])
await assert_video_search(search_str='5', deep=True, assert_ids=[5, ])
# only video 1 has e and d
await assert_video_search(search_str='e d', assert_ids=[1, ])
await assert_video_search(search_str='e d', deep=True, assert_ids=[1, ])
# video 1 and 4 have b and e, but 1 has more
await assert_video_search(search_str='b e', assert_ids=[1, 4])
await assert_video_search(search_str='b e', deep=True, assert_ids=[1, 4])
# Check totals are correct even with a limit
await assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
await assert_video_search(search_str='b', deep=True, limit=2, assert_ids=[1, 2], assert_total=4)
@pytest.mark.asyncio

2
modules/videos/test/test_downloader.py
View file

@ -16,6 +16,7 @@ from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
from wrolpi.downloader import Download, DownloadResult, get_download_manager_config
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
from wrolpi.test.common import skip_circleci
from wrolpi.vars import PROJECT_DIR
example_video_json = {
@ -130,6 +131,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
assert [i.tag.name for i in video.file_group.tag_files] == [tag1.name, tag2.name]
@skip_circleci
@pytest.mark.asyncio
async def test_download_channel(test_session, test_directory, simple_channel, video_download_manager, video_file,
mock_video_extract_info, mock_video_prepare_filename, await_switches,

1
modules/videos/video/api.py
View file

@ -66,6 +66,7 @@ async def search_videos(_: Request, body: schema.VideoSearchRequest):
body.tag_names,
body.headline,
body.censored,
body.deep,
)
ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}}

7
modules/videos/video/lib.py
View file

@ -93,6 +93,7 @@ def search_videos(
tag_names: List[str] = None,
headline: bool = False,
censored: bool = False,
deep: bool = False,
) -> Tuple[List[dict], int]:
tag_names = tag_names or []
# Search videos and audio-only files.
@ -112,8 +113,10 @@ def search_videos(
if search_str:
# A search_str was provided by the user, modify the query to filter by it.
select_columns = 'fg.id, ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
# `textsearch` includes d_text (captions); `textsearch_abc` is much smaller and faster.
ts_col = 'textsearch' if deep else 'textsearch_abc'
select_columns = f'fg.id, ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
params['search_str'] = search_str
else:
# No search_str provided. Get id and total only.

4
scripts/wrolpi-usb.sh
View file

@ -165,7 +165,7 @@ ERROR: the new ISO is larger than the headroom on this USB.
To upgrade, you'll need to:
1. Back up /media/wrolpi/ to another drive.
2. Re-flash the new ISO from scratch with dd or Etcher.
2. Re-flash the new ISO from scratch with dd or Raspberry Pi Imager.
3. Restore /media/wrolpi/ from the backup.
Future versions flashed from a release ≥ 0.22 reserve 8 GiB of
@ -225,7 +225,7 @@ WROLPi USB without losing your files:
1. Back up everything on the drive to another disk.
2. Flash the whole drive with the ISO (this erases it):
sudo dd if=<iso> of=<device> bs=4M status=progress conv=fsync
(or use Etcher / Rufus / Raspberry Pi Imager)
(or use Raspberry Pi Imager / Rufus)
3. Boot the drive once. WROLPi creates its persistence partition and
sets itself up on first boot.
4. Copy your backed-up data into /media/wrolpi/ on the running system,

2
wrolpi/files/api.py
View file

@ -165,7 +165,7 @@ async def post_search_files(_: Request, body: schema.FilesSearchRequest):
with timer('Searching all files', 'info', logger__=logger):
file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
body.tag_names, body.headline, body.months, body.from_year, body.to_year,
body.any_tag, body.order, body.url, body.suffix, body.path)
body.any_tag, body.order, body.url, body.suffix, body.path, body.deep)
return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))

31
wrolpi/files/lib.py
View file

@ -1039,7 +1039,7 @@ async def search_directories_by_name(session: Session, name: str, excluded: List
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None,
url: str = None, suffix: str = None, path: str = None) -> \
url: str = None, suffix: str = None, path: str = None, deep: bool = False) -> \
Tuple[List[dict], int]:
"""Search the FileGroup table.
@ -1061,6 +1061,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
@param url: Filter by URL using case-insensitive partial match (ILIKE).
@param suffix: Only return files whose primary file has this suffix (e.g. ".bin"), case-insensitive.
@param path: Filter by primary_path using case-insensitive partial match (ILIKE).
@param deep: Search all text including captions/body (d_text); slower but more thorough.
"""
params = dict(offset=offset, limit=limit, search_str=search_str, url_search_str=f'%{search_str}%')
wheres = []
@ -1069,9 +1070,10 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
joins = []
if search_str:
# Search by textsearch column.
wheres.append('textsearch @@ websearch_to_tsquery(%(search_str)s)')
selects.append('ts_rank(textsearch, websearch_to_tsquery(%(search_str)s))')
# `textsearch` includes d_text (captions/body); `textsearch_abc` is much smaller and faster.
ts_col = 'textsearch' if deep else 'textsearch_abc'
wheres.append(f'{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
selects.append(f'ts_rank({ts_col}, websearch_to_tsquery(%(search_str)s))')
order_by = '2 DESC'
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
@ -1479,9 +1481,13 @@ def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str
async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str],
months: List[int] = None, from_year: int = None, to_year: int = None,
any_tag: bool = False):
any_tag: bool = False) -> dict:
"""
Return FileGroup count of what will be returned if the search is actually performed.
Return FileGroup counts of what will be returned if the search is actually performed.
Returns both the fast (`textsearch_abc`) and deep (`textsearch`) counts so the UI can show the count
matching the current search mode, and offer deep search when it has more results. Both are cheap
index-bound COUNTs (no ts_rank).
"""
wheres = []
joins = list()
@ -1490,8 +1496,13 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
having = ''
if search_str:
# Search by textsearch, and by matching url.
# `textsearch` matches are a superset of `textsearch_abc`, so filter by the deep column and count
# the fast subset with FILTER in the same pass.
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
selects = ('COUNT(*) FILTER (WHERE fg.textsearch_abc @@ websearch_to_tsquery(%(search_str)s)) AS estimate,'
' COUNT(*) AS deep_estimate')
else:
selects = 'COUNT(*) AS estimate, COUNT(*) AS deep_estimate'
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
@ -1501,7 +1512,7 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
joins = "\n".join(joins)
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
stmt = f'''
SELECT COUNT(*) OVER() AS estimate
SELECT {selects}
FROM file_group fg
{joins}
{wheres}
@ -1514,8 +1525,8 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
curs.execute(stmt, params)
result = curs.fetchone()
if result:
return int(result['estimate'])
return 0
return dict(file_groups=int(result['estimate']), file_groups_deep=int(result['deep_estimate']))
return dict(file_groups=0, file_groups_deep=0)
MOVE_CHUNK_SIZE = 100

8
wrolpi/files/models.py
View file

@ -79,6 +79,7 @@ class FileGroup(ModelHelper, Base):
Index('file_group_published_datetime_idx', 'published_datetime'),
Index('file_group_published_modified_datetime_idx', 'published_modified_datetime'),
Index('file_group_size_ix', 'size'),
Index('file_group_textsearch_abc_idx', 'textsearch_abc'),
Index('file_group_textsearch_idx', 'textsearch'),
Index('file_group_url_idx', 'url'),
Index('file_group_viewed_idx', 'viewed'),
@ -126,6 +127,13 @@ class FileGroup(ModelHelper, Base):
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char")
''')))
# Excludes d_text (captions/body); much smaller than `textsearch`, used for the default fast search.
textsearch_abc = deferred(
Column(tsvector, Computed('''
setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char")
''')))
def __repr__(self):
m = f'model={self.model}' if self.model else f'mimetype={self.mimetype}'

1
wrolpi/files/schema.py
View file

@ -44,6 +44,7 @@ class FilesSearchRequest:
url: Optional[str] = None # Filter by URL (partial match)
suffix: Optional[str] = None # Filter by primary file suffix (e.g. ".bin"), case-insensitive exact match
path: Optional[str] = None # Filter by primary_path (case-insensitive partial match)
deep: bool = False # When True, search all text including captions/body (d_text)
def __post_init__(self):
if self.any_tag and self.tag_names:

48
wrolpi/files/test/test_api.py
View file

@ -315,6 +315,51 @@ async def test_files_search(test_session, async_client, make_files_structure, as
await assert_files_search('nothing', [])
@pytest.mark.asyncio
async def test_files_search_deep(test_session, async_client, make_files_structure):
"""Search defaults to the fast ABC column (title/author/description); `deep` opts into d_text
(captions/file contents)."""
guide, lecture = make_files_structure([
'gardening guide.txt',
'lecture.txt',
])
guide.write_text('planting tips')
# "gardening" only appears in this file's contents (d_text), not its name.
lecture.write_text('a lecture about gardening')
request, response = await async_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
async def search(**kwargs):
request, response = await async_client.post('/api/files/search', json=kwargs)
assert response.status_code == HTTPStatus.OK
return response.json['file_groups']
# Default (fast) search only matches the title.
results = await search(search_str='gardening')
assert [i['primary_path'] for i in results] == ['gardening guide.txt']
# Deep search also matches file contents; the title match ranks higher (weight A vs D).
results = await search(search_str='gardening', deep=True)
assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
# Headlines work in both modes.
results = await search(search_str='gardening', headline=True)
assert [i['primary_path'] for i in results] == ['gardening guide.txt']
assert '<b>gardening</b>' in results[0]['title_headline']
results = await search(search_str='gardening', deep=True, headline=True)
assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
assert '<b>gardening</b>' in results[1]['d_headline']
# The estimate endpoint returns both counts: `file_groups` matches the default fast search, and
# `file_groups_deep` shows what deep search would find, so the UI can offer deep search.
request, response = await async_client.post('/api/search_file_estimates',
json=dict(search_str='gardening'))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == 1
assert response.json['file_groups_deep'] == 2
@pytest.mark.asyncio
async def test_files_search_by_suffix(test_session, async_client, make_files_structure):
"""Files can be filtered by their primary file's suffix (used by the firmware flasher to find .bin files)."""
@ -367,7 +412,8 @@ async def test_files_search_attaches_doc_section_hint(async_client, test_session
bug where only the docs-specific endpoint (/api/docs/search) attached hints."""
await refresh_files()
body = dict(search_str='chapter')
# "chapter" only appears in the EPUB body (d_text), so deep search is required to match it.
body = dict(search_str='chapter', deep=True)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK

12
wrolpi/files/test/test_lib.py
View file

@ -580,18 +580,20 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
# Video are indexed by the modeler, not by an indexer.
assert video_file.mimetype == 'video/mp4' and video_file.indexer == indexers.DefaultIndexer
# File are indexed by their titles and contents.
# File are indexed by their titles and contents. Contents (d_text) require a deep search.
files, total = lib.search_files('file', 10, 0)
assert total == 5, 'All files contain "file" in their file name. The associated video file is hidden.'
files, total = lib.search_files('image', 10, 0)
assert total == 1 and files[0]['title'] == 'an image file.jpeg', 'The image file title contains "image".'
files, total = lib.search_files('contents', 10, 0)
files, total = lib.search_files('contents', 10, 0, deep=True)
assert total == 1 and files[0]['title'] == 'a text file.txt', 'The text file contains "contents".'
files, total = lib.search_files('contents', 10, 0)
assert total == 0, 'File contents are not searched unless deep=True.'
files, total = lib.search_files('video', 10, 0)
assert total == 1 and {i['title'] for i in files} == {'a video file'}, 'The video file contains "video".'
files, total = lib.search_files('yawn', 10, 0)
files, total = lib.search_files('yawn', 10, 0, deep=True)
assert total == 1 and files[0]['title'] == 'a video file', 'The video file captions contain "yawn".'
files, total = lib.search_files('bunny', 10, 0)
files, total = lib.search_files('bunny', 10, 0, deep=True)
assert total == 1 and {i['title'] for i in files} == {'a zip file.zip'}, \
'The zip file contains a file with "bunny" in the title.'
@ -603,7 +605,7 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
text_path.write_text('new text contents')
await refresh_files()
await await_background_tasks()
files, total = lib.search_files('new', 10, 0)
files, total = lib.search_files('new', 10, 0, deep=True)
assert total == 1

5
wrolpi/root_api.py
View file

@ -768,7 +768,7 @@ async def post_search_suggestions(request: Request, body: schema.SearchSuggestio
)
async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimateRequest):
"""Used by the Global search to suggest FileGroup count to the user."""
file_groups = await search_file_suggestion_count(
counts = await search_file_suggestion_count(
body.search_str,
body.tag_names,
body.mimetypes,
@ -779,7 +779,8 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate
)
ret = dict(
file_groups=file_groups,
file_groups=counts['file_groups'],
file_groups_deep=counts['file_groups_deep'],
)
return json_response(ret)

6
wrolpi/test/test_root_api.py
View file

@ -666,7 +666,8 @@ async def test_search_file_estimates(test_session, async_client, archive_factory
request, response = await async_client.post('/api/search_file_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == expected_file_groups or 0
# The deep count includes contents/captions matches; the fast (ABC) count is in `file_groups`.
assert response.json['file_groups_deep'] == expected_file_groups or 0
await assert_results(dict(search_str='foo'), 1)
@ -727,7 +728,8 @@ async def test_search_file_estimates_any_tag(test_session, async_client, archive
request, response = await async_client.post('/api/search_file_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == expected_file_groups or 0
# The deep count includes contents/captions matches; the fast (ABC) count is in `file_groups`.
assert response.json['file_groups_deep'] == expected_file_groups or 0
await assert_results(dict(search_str='bunny'), 2)
await assert_results(dict(search_str='bunny', any_tag=True), 1)

2
wrolpi/version.txt
View file

@ -1 +1 @@
0.26.0-beta
0.27.0-beta