Other search tab: search Channels by name, add Domains and Playlists, fix the estimate (#785)
Some checks are pending
CodeQL / Analyze (push) Waiting to run
Controller Tests / controller-tests (push) Waiting to run
Jest Tests / jest-tests (push) Waiting to run

* Count Channels with any of the Tags in the Other search estimate

The Other tab lists Channels with any of the selected Tags, but its estimate
only counted Channels with the first Tag, so the two disagreed when several
Tags were selected.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Search Channels by name in the Other search tab

The Other tab only listed Channels with the selected Tags, so typing a
Channel's name found nothing.  Channels now match the search text by name,
and the Tags when there are both.  The list and its estimate share one query.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Add Domains to the Other search tab

Domains match the search text by name and/or any of the Tags, like Channels.
The list and its estimate share one query.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Add Playlists to the Other search tab

Domains and Playlists are Collections, so the Other tab searches both with
the existing /api/collections/search (by kind) instead of a Domain-only
endpoint.  The search and the estimate share search_collections_query.  The
search now loads each Collection's Downloads up front instead of once per
Collection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-07 18:22:05 -06:00 • committed by Roland
parent 93d878952c
commit a1292005ce
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
14 changed files with 309 additions and 73 deletions

View file

@ -2118,8 +2118,8 @@ export async function searchEstimateOthers(tagNames, searchStr) {
}
}
export async function searchChannels(tagNames) {
const body = {tag_names: tagNames};
export async function searchChannels(tagNames, searchStr) {
const body = {tag_names: tagNames, search_str: searchStr};
const response = await apiPost(`${API_URI}/videos/channels/search`, body);
if (response.ok) {
const content = await response.json();
@ -2129,6 +2129,16 @@ export async function searchChannels(tagNames) {
}
}
// Collections of `kind` whose name matches `searchStr` and/or which have any of the tags.
export async function searchCollections(kind, tagNames, searchStr) {
const body = {kind, tag_names: tagNames, search_str: searchStr};
const response = await apiPost(`${COLLECTIONS_API}/search`, body);
if (response.ok) {
const content = await response.json();
return {collections: content.collections};
}
}
export async function getOutdatedZims() {
const response = await apiGet(`${API_URI}/zim/outdated`);
if (response.status === 200) {

View file

@ -1,7 +1,7 @@
import React from 'react';
import {act, render, screen} from '../test-utils';
import {OtherSearchView} from './Search';
import {searchChannels, searchRepos} from '../api';
import {searchChannels, searchCollections, searchRepos} from '../api';
// What the search page's Other tab shows for each state of useSearchChannels. "No Channels"
// must only appear once the server has confirmed there are none.
@ -9,6 +9,7 @@ import {searchChannels, searchRepos} from '../api';
jest.mock('../api', () => ({
...jest.requireActual('../api'),
searchChannels: jest.fn(),
searchCollections: jest.fn(),
searchRepos: jest.fn(),
}));
@ -18,6 +19,8 @@ beforeEach(() => {
searchChannels.mockReset();
searchRepos.mockReset();
searchRepos.mockResolvedValue({repos: [], total: 0});
searchCollections.mockReset();
searchCollections.mockResolvedValue({collections: []});
});
test('pending: shows a loader, not "No Channels"', () => {
@ -117,3 +120,55 @@ describe('Repos', () => {
expect(screen.getByText(/files <script>/)).toBeInTheDocument();
});
});
// Domains and Playlists are Collections searched by kind.
describe.each([
['domain', 'Domains', 'example.com', '/archives?domain=example.com'],
['playlist', 'Playlists', 'Road Trip', '/playlists/3'],
])('%s Collections', (kind, title, name, href) => {
const collectionsOf = (wanted, collections) => (k) => Promise.resolve({collections: k === wanted ? collections : []});
test(`empty: shows "No ${title}"`, async () => {
searchChannels.mockResolvedValue({channels: []});
render(<OtherSearchView loading={false}/>, {query: {searchParams: new URLSearchParams('q=nothing')}});
await act(async () => {
});
expect(screen.getByText(`No ${title}`)).toBeInTheDocument();
});
test('nothing to search by: shows none without fetching', async () => {
searchChannels.mockResolvedValue({channels: []});
render(<OtherSearchView loading={false}/>);
await act(async () => {
});
expect(screen.getByText(`No ${title}`)).toBeInTheDocument();
expect(searchCollections).not.toHaveBeenCalled();
});
test(`failed: shows an error only in the ${title} section`, async () => {
searchChannels.mockResolvedValue({channels: [{id: 1, name: 'Cooking'}]});
searchCollections.mockImplementation(k => k === kind ? Promise.reject(new Error('boom')) : Promise.resolve({collections: []}));
const spy = jest.spyOn(console, 'error').mockImplementation(() => {
});
try {
render(<OtherSearchView loading={false}/>, {query: {searchParams: new URLSearchParams('q=x')}});
await act(async () => {
});
expect(screen.getByText('Cooking')).toBeInTheDocument();
expect(screen.getByText(new RegExp(`Could not fetch the ${title.toLowerCase()}`))).toBeInTheDocument();
} finally {
spy.mockRestore();
}
});
test('loaded: lists them by the search and tags, linked', async () => {
searchChannels.mockResolvedValue({channels: []});
searchCollections.mockImplementation(collectionsOf(kind, [{id: 3, name, kind, tag_name: null}]));
const searchParams = new URLSearchParams('q=road&tag=News');
render(<OtherSearchView loading={false}/>, {query: {searchParams}});
await act(async () => {
});
expect(searchCollections).toHaveBeenCalledWith(kind, ['News'], 'road');
expect(screen.getByText(name).closest('a')).toHaveAttribute('href', href);
});
});

View file

@ -8,6 +8,7 @@ import {
useSearchDate,
useSearchDeep,
useSearchFilter,
useSearchCollections,
useSearchRepos,
} from "../hooks/customHooks";
import {ShortcutHint} from "./ShortcutHint";
@ -562,6 +563,16 @@ function SearchChannelPreview({channel}) {
</div>
}
function SearchCollectionPreview({collection, to}) {
const {SingleTag} = React.useContext(TagsContext);
return <div>
<span style={{display: 'inline-flex', alignItems: 'center', gap: 8, flexWrap: 'wrap'}}>
<Link to={to}>{collection.name}</Link>
{collection.tag_name && <SingleTag name={collection.tag_name}/>}
</span>
</div>
}
function SearchRepoPreview({repo}) {
const {SingleTag} = React.useContext(TagsContext);
return <div>
@ -594,14 +605,16 @@ function OtherSearchSection({value, title, items, preview, emptyText, errorText}
</Accordion.Item>
}
// `channels` follows useSearchChannels and `repos` follows useSearchRepos: null = pending,
// Each section's items follow useSearchChannels, useSearchCollections, or useSearchRepos: null = pending,
// undefined = fetch failed, [] = none.
export function OtherSearchView({loading}) {
const {searchParams} = React.useContext(QueryContext);
const [activeValues, setActiveValues] = React.useState(['channels', 'repos']);
const [activeValues, setActiveValues] = React.useState(['channels', 'domains', 'playlists', 'repos']);
const activeTags = searchParams.getAll('tag');
const searchStr = searchParams.get('q') || '';
const {channels, loading: channelsLoading} = useSearchChannels(activeTags);
const {channels, loading: channelsLoading} = useSearchChannels(activeTags, searchStr);
const {collections: domains} = useSearchCollections('domain', searchStr, activeTags);
const {collections: playlists} = useSearchCollections('playlist', searchStr, activeTags);
const {repos} = useSearchRepos(searchStr, activeTags);
if (loading || channelsLoading || channels === null) {
@ -619,6 +632,23 @@ export function OtherSearchView({loading}) {
emptyText='No Channels'
errorText='Could not fetch the channels.'
/>
<OtherSearchSection
value='domains'
title='Domains'
items={domains}
preview={i => <SearchCollectionPreview key={i.id} collection={i}
to={`/archives?domain=${encodeURIComponent(i.name)}`}/>}
emptyText='No Domains'
errorText='Could not fetch the domains.'
/>
<OtherSearchSection
value='playlists'
title='Playlists'
items={playlists}
preview={i => <SearchCollectionPreview key={i.id} collection={i} to={`/playlists/${i.id}`}/>}
emptyText='No Playlists'
errorText='Could not fetch the playlists.'
/>
<OtherSearchSection
value='repos'
title='Repos'

View file

@ -36,6 +36,7 @@ import {
saveSettings,
searchArchives,
searchChannels,
searchCollections,
searchDirectories,
searchDocs,
searchRepos,
@ -2107,7 +2108,8 @@ export const useCalcQuery = () => {
return [calc, setCalc]
}
export const useSearchChannels = (defaultTagNames) => {
// Channels whose name matches `searchStr` and/or which have any of the tags.
export const useSearchChannels = (defaultTagNames, searchStr) => {
const [tagNames, setTagNames] = useState(defaultTagNames || []);
// null = pending, undefined = fetch failed, [] = no channels. A search starts on mount, so the
// first render is already loading; consumers must never see [] for a search that has not run.
@ -2121,7 +2123,7 @@ export const useSearchChannels = (defaultTagNames) => {
setChannels(null);
setLoading(true);
try {
const {channels: newChannels} = await searchChannels(tagNames);
const {channels: newChannels} = await searchChannels(tagNames, searchStr);
if (gen === requestGen.current) {
setChannels(newChannels || []);
}
@ -2147,7 +2149,7 @@ export const useSearchChannels = (defaultTagNames) => {
useEffect(() => {
localSearchChannels();
}, [tagNames]);
}, [tagNames, searchStr]);
return {
tagNames,
@ -2157,6 +2159,33 @@ export const useSearchChannels = (defaultTagNames) => {
}
}
// Collections of `kind` whose name matches `searchStr` and/or which have any of the tags. `collections`:
// null = pending, undefined = fetch failed, [] = none. Nothing to search by matches nothing (the API would list
// every Collection of the kind). Only the newest request may write state.
export const useSearchCollections = (kind, searchStr, tagNames) => {
const [collections, setCollections] = useState(null);
const requestGen = useRef(0);
const tagKey = JSON.stringify(tagNames || []);
useEffect(() => {
const gen = ++requestGen.current;
const tags = JSON.parse(tagKey);
if (!(searchStr || '').trim() && _.isEmpty(tags)) {
setCollections([]);
return;
}
setCollections(null);
searchCollections(kind, tags, searchStr)
.then(result => gen === requestGen.current && setCollections(result ? result.collections : undefined))
.catch(e => {
console.error(e);
if (gen === requestGen.current) setCollections(undefined);
});
}, [kind, searchStr, tagKey]);
return {collections};
}
// Repos matching the search (their name and README) and/or the tags. `repos`: null = pending,
// undefined = fetch failed, [] = none. Only the newest request may write state.
export const useSearchRepos = (searchStr, tagNames) => {

View file

@ -475,7 +475,7 @@ describe('useSearchChannels', () => {
const {result} = renderHook(() => useSearchChannels(['a']));
await act(async () => {
});
expect(searchChannels).toHaveBeenCalledWith(['a']);
expect(searchChannels).toHaveBeenCalledWith(['a'], undefined);
expect(result.current.channels).toEqual([{id: 1, name: 'One'}]);
expect(result.current.loading).toBe(false);
});
@ -510,11 +510,27 @@ describe('useSearchChannels', () => {
}));
rerender({tags: ['new']});
expect(searchChannels).toHaveBeenCalledTimes(2);
expect(searchChannels).toHaveBeenLastCalledWith(['new']);
expect(searchChannels).toHaveBeenLastCalledWith(['new'], undefined);
expect(result.current.channels).toBeNull();
expect(result.current.loading).toBe(true);
});
test('searches by the search text, and refetches when it changes', async () => {
searchChannels.mockResolvedValue({channels: [{id: 1, name: 'Cooking'}]});
const {result, rerender} = renderHook(({q}) => useSearchChannels([], q), {initialProps: {q: 'cook'}});
await act(async () => {
});
expect(searchChannels).toHaveBeenLastCalledWith([], 'cook');
expect(result.current.channels).toEqual([{id: 1, name: 'Cooking'}]);
searchChannels.mockReturnValue(new Promise(() => {
}));
rerender({q: 'garden'});
expect(searchChannels).toHaveBeenCalledTimes(2);
expect(searchChannels).toHaveBeenLastCalledWith([], 'garden');
expect(result.current.channels).toBeNull();
});
test('a new array with the same tags does not refetch', async () => {
searchChannels.mockResolvedValue({channels: []});
const {rerender} = renderHook(({tags}) => useSearchChannels(tags), {initialProps: {tags: ['a']}});

View file

@ -116,13 +116,13 @@ async def test_search_api(test_session, async_client, test_wrolpi_config):
# The global search's "Other" estimate counts Repos by the search, and by Tag.
request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
assert response.json['others'] == dict(channel_count=0, repo_count=1)
assert response.json['others'] == dict(channel_count=0, domain_count=0, playlist_count=0, repo_count=1)
body = dict(tag_names=['software'])
request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
assert response.json['others'] == dict(channel_count=0, repo_count=1)
assert response.json['others'] == dict(channel_count=0, domain_count=0, playlist_count=0, repo_count=1)
body = dict(tag_names=['software'], search_str='nothing')
request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
assert response.json['others'] == dict(channel_count=0, repo_count=0)
assert response.json['others'] == dict(channel_count=0, domain_count=0, playlist_count=0, repo_count=0)
# Repos are suggested by name.
request, response = await async_client.post('/api/search_suggestions', content=json.dumps(dict(search_str='kiw')))

View file

@ -145,7 +145,7 @@ async def channel_tag(request: Request, channel_id: int, body: schema.ChannelTag
@validate(schema.ChannelSearchRequest)
async def channel_search(request: Request, body: schema.ChannelSearchRequest):
session = request.ctx.session
channels = await lib.search_channels(session, body.tag_names)
channels = await lib.search_channels(session, body.tag_names, body.search_str)
ret = dict(
channels=channels,
)

View file

@ -1,9 +1,9 @@
import pathlib
from pathlib import Path
from typing import List, Union
from typing import List, Optional, Union
from sqlalchemy import or_, func, desc
from sqlalchemy.orm import Session, joinedload
from sqlalchemy.orm import Query, Session, joinedload
from wrolpi import flags
from wrolpi.collections import Collection
@ -330,8 +330,27 @@ async def tag_channel(session: Session, tag_name: str | None, directory: pathlib
save_channels_config.activate_switch()
async def search_channels(session: Session, tag_names: List[str]) -> List[Channel]:
"""Search Tagged Channels."""
channels = session.query(Channel).options(*Channel.json_options()) \
.join(Collection).join(Tag).filter(Tag.name.in_(tag_names)).all()
return channels
def search_channels_query(session: Session, tag_names: List[str] = None, search_str: str = None) -> Optional[Query]:
"""Channels whose name contains `search_str` and which have any of the Tags. None when there is nothing to search
by."""
search_str = (search_str or '').strip()
if not tag_names and not search_str:
return None
query = session.query(Channel).join(Collection, Collection.id == Channel.collection_id)
if search_str:
name_no_spaces = ''.join(search_str.split(' '))
query = query.filter(or_(
Collection.name.ilike(f'%{search_str}%'),
Collection.name.ilike(f'%{name_no_spaces}%'),
))
if tag_names:
query = query.join(Tag, Tag.id == Collection.tag_id).filter(Tag.name.in_(tag_names))
return query
async def search_channels(session: Session, tag_names: List[str] = None, search_str: str = None) -> List[Channel]:
"""Search Channels by their name and/or Tags."""
query = search_channels_query(session, tag_names, search_str)
if query is None:
return []
return query.options(*Channel.json_options()).order_by(func.lower(Collection.name)).all()

View file

@ -721,3 +721,32 @@ async def test_search_tagged_channels(async_client, test_session, channel_factor
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 1, 'Only one Channel is tagged'
assert response.json['channels'][0]['id'] == channel1.id
@pytest.mark.asyncio
async def test_search_channels_by_name(async_client, test_session, channel_factory, tag_factory):
"""Channels can be searched by their name, alone or with Tags."""
tag = await tag_factory()
cooking = channel_factory(name='Cooking Show', tag_name=tag.name)
channel_factory(name='Cooking Basics')
channel_factory(name='Gardening')
test_session.commit()
async def search(body):
request, response = await async_client.post('/api/videos/channels/search', json=body)
assert response.status_code == HTTPStatus.OK
return sorted(i['name'] for i in response.json['channels'])
assert await search(dict(search_str='cooking')) == ['Cooking Basics', 'Cooking Show']
# The name and the Tags must both match.
assert await search(dict(search_str='cooking', tag_names=[tag.name])) == ['Cooking Show']
assert await search(dict(search_str='gardening', tag_names=[tag.name])) == []
# Nothing to search by matches nothing.
assert await search(dict(search_str='')) == []
assert await search(dict()) == []
# The Other tab's estimate counts the same Channels.
for body in (dict(search_str='cooking'), dict(search_str='cooking', tag_names=[tag.name]), dict(search_str='')):
request, response = await async_client.post('/api/search_other_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['others']['channel_count'] == len(await search(body)), body

View file

@ -266,6 +266,7 @@ class CensoredVideoResponse:
@dataclass
class ChannelSearchRequest:
tag_names: List[str] = field(default_factory=list)
search_str: Optional[str] = None
@dataclass

View file

@ -8,7 +8,7 @@ import pathlib
from typing import List, Optional, Dict
from sqlalchemy import func
from sqlalchemy.orm import Session, selectinload, noload
from sqlalchemy.orm import Query, Session, selectinload, noload
from wrolpi import flags
from wrolpi.common import get_media_directory, logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
@ -871,42 +871,35 @@ def delete_collection(
return collection_dict
def search_collections_query(
session: Session,
kind: Optional[str] = None,
tag_names: Optional[List[str]] = None,
search_str: Optional[str] = None,
) -> Query:
"""Collections of `kind` whose name contains `search_str` and which have any of the Tags. A missing filter
matches every Collection."""
query = session.query(Collection)
if kind:
query = query.filter(Collection.kind == kind)
if tag_names:
query = query.join(Tag, Tag.id == Collection.tag_id).filter(Tag.name.in_(tag_names))
if search_str:
query = query.filter(Collection.name.ilike(f'%{search_str}%'))
return query
def search_collections(
session: Session,
kind: Optional[str] = None,
tag_names: Optional[List[str]] = None,
search_str: Optional[str] = None,
) -> List[dict]:
"""
Search collections by kind, tags, and search string.
Args:
session: Database session
kind: Optional collection kind filter
tag_names: Optional list of tag names to filter by
search_str: Optional search string for collection names
Returns:
List of matching collection dicts
"""
query = session.query(Collection)
# Filter by kind
if kind:
query = query.filter(Collection.kind == kind)
# Filter by tags
if tag_names:
query = query.join(Tag).filter(Tag.name.in_(tag_names))
# Filter by search string
if search_str:
query = query.filter(Collection.name.ilike(f'%{search_str}%'))
# Order by name
query = query.order_by(Collection.name)
collections = query.all()
"""Search Collections by kind, Tags, and name (see `search_collections_query`)."""
collections = search_collections_query(session, kind, tag_names, search_str) \
.options(*Collection.json_options()) \
.order_by(Collection.name) \
.all()
return [collection.__json__() for collection in collections]

View file

@ -0,0 +1,47 @@
from http import HTTPStatus
import pytest
from wrolpi.collections import Collection
@pytest.mark.asyncio
@pytest.mark.parametrize('kind', ['domain', 'playlist'])
async def test_search_collections_other_estimate(async_client, test_session, tag_factory, kind):
"""Collections can be searched by kind, name, and/or Tags; the Other tab's estimate counts the same
Collections."""
tag1, tag2 = await tag_factory(), await tag_factory()
test_session.add_all([
Collection(name='cooking one', kind=kind, tag=tag1),
Collection(name='cooking two', kind=kind),
Collection(name='gardening', kind=kind, tag=tag2),
# Another kind is never counted.
Collection(name='cooking other', kind='channel' if kind == 'domain' else 'domain', tag=tag1),
])
test_session.commit()
async def search(body):
request, response = await async_client.post('/api/collections/search', json=dict(body, kind=kind))
assert response.status_code == HTTPStatus.OK
return sorted(i['name'] for i in response.json['collections'])
cases = [
(dict(search_str='cooking'), ['cooking one', 'cooking two']),
(dict(tag_names=[tag1.name]), ['cooking one']),
# Any of the Tags.
(dict(tag_names=[tag1.name, tag2.name]), ['cooking one', 'gardening']),
# The name and the Tags must both match.
(dict(search_str='garden', tag_names=[tag2.name]), ['gardening']),
(dict(search_str='cooking', tag_names=[tag2.name]), []),
]
count_key = f'{kind}_count'
for body, expected in cases:
assert await search(body) == expected, body
request, response = await async_client.post('/api/search_other_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['others'][count_key] == len(expected), body
# Nothing to search by: the Other tab shows none (the search endpoint lists every Collection of the kind).
for body in (dict(search_str=''), dict(search_str=' '), dict()):
request, response = await async_client.post('/api/search_other_estimates', json=body)
assert response.json['others'][count_key] == 0, body

View file

@ -2870,31 +2870,26 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
async def search_other_estimates(session: Session, tag_names: List[str], search_str: str = None) -> dict:
"""Estimate other things that are Tagged, or match the search (Repos by their name and README)."""
from sqlalchemy import func
from wrolpi.collections.models import Collection
from wrolpi.tags import Tag
from modules.videos.models import Channel
"""Estimate other things that match the search or the Tags (Channels, Domains, and Playlists by their name, Repos
by their name and README)."""
from modules.videos.channel.lib import search_channels_query
from modules.repos.lib import count_repos
from wrolpi.collections.lib import search_collections_query
repo_count = count_repos(session, search_str, tag_names)
search_str = (search_str or '').strip()
channels = search_channels_query(session, tag_names, search_str)
if not tag_names:
return dict(
channel_count=0,
repo_count=repo_count,
)
# TODO handle multiple tags
channel_count = session.query(func.count(Channel.id)) \
.join(Collection, Collection.id == Channel.collection_id) \
.join(Tag, Tag.id == Collection.tag_id) \
.filter(Tag.name == tag_names[0]) \
.scalar()
def count_collections(kind: str) -> int:
# Nothing to search by matches nothing, like the other searches.
if not tag_names and not search_str:
return 0
return search_collections_query(session, kind, tag_names, search_str).count()
others = dict(
channel_count=channel_count,
repo_count=repo_count,
channel_count=channels.count() if channels is not None else 0,
domain_count=count_collections('domain'),
playlist_count=count_collections('playlist'),
repo_count=count_repos(session, search_str, tag_names),
)
return others

View file

@ -1016,6 +1016,18 @@ async def test_search_other_estimates(async_client, test_session, channel_factor
assert response.status_code == HTTPStatus.OK
assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.'
# The estimate counts the Channels the search lists: those with any of the Tags.
tag2 = await tag_factory()
channel_factory(tag_name=tag2.name)
test_session.commit()
body = dict(tag_names=[tag.name, tag2.name])
request, response = await async_client.post('/api/search_other_estimates', json=body)
assert response.status_code == HTTPStatus.OK
estimate = response.json['others']['channel_count']
request, response = await async_client.post('/api/videos/channels/search', json=body)
assert response.status_code == HTTPStatus.OK
assert estimate == len(response.json['channels']) == 2
@pytest.mark.asyncio
async def test_mutating_request_survives_concurrent_writer(async_client, test_session, test_downloader,