The Any Tag filter works on the Videos, Archives and Docs searches (#958)

The filter modal on these pages offered "Any Tag", but only the Files search
sent it; the module searches had no `any_tag` field.  Docs also matched any of
the listed tag_names, while every other search requires all of them.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-11 19:21:06 -07:00 • committed by GitHub
parent a0f30a0a60
commit 2800112f3e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 189 additions and 30 deletions

View file

@ -196,7 +196,7 @@ export async function tagChannelInfo(channelId, tagName) {
});
}
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline, censored, deep) {
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline, censored, deep, anyTag) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@ -220,6 +220,9 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
if (deep) {
body['deep'] = true;
}
if (anyTag) {
body['any_tag'] = true;
}
console.debug('searching videos', body);
@ -852,7 +855,7 @@ export async function saveCatalog(items) {
}
export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline, deep) {
export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline, deep, anyTag) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@ -875,6 +878,9 @@ export async function searchArchives(offset, limit, domain, searchStr, order, ta
if (deep) {
body['deep'] = true;
}
if (anyTag) {
body['any_tag'] = true;
}
console.debug('searching archives', body);
const response = await apiPost(`${ARCHIVES_API}/search`, body);
@ -2438,7 +2444,7 @@ export async function getDoc(fileGroupId) {
}
export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject, mimetype, deep) {
export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject, mimetype, deep, anyTag) {
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
let body = {offset, limit};
@ -2463,6 +2469,9 @@ export async function searchDocs(offset, limit, searchStr, order, tagNames, auth
if (deep) {
body['deep'] = true;
}
if (anyTag) {
body['any_tag'] = true;
}
console.debug('searching docs', body);
const response = await apiPost(`${DOCS_API}/search`, body);

View file

@ -543,6 +543,7 @@ export const useSearchArchives = (defaultLimit) => {
const searchStr = searchParams.get('q') || '';
const order = searchParams.get('order');
const activeTags = searchParams.getAll('tag');
const anyTag = searchParams.get('anyTag') === 'true';
const {view} = useSearchView();
const headline = view === 'headline';
const {deep} = useSearchDeep();
@ -554,7 +555,7 @@ export const useSearchArchives = (defaultLimit) => {
setArchives(null);
try {
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline,
deep);
deep, anyTag);
setArchives(archives_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -571,7 +572,7 @@ export const useSearchArchives = (defaultLimit) => {
useEffect(() => {
localSearchArchives();
}, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), headline, deep]);
}, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), anyTag, headline, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -615,6 +616,7 @@ export const useSearchDocs = (defaultLimit) => {
const author = searchParams.get('author') || '';
const subject = searchParams.get('subject') || '';
const filter = searchParams.get('filter') || '';
const anyTag = searchParams.get('anyTag') === 'true';
const {deep} = useSearchDeep();
const [docs, setDocs] = useState(null);
@ -625,7 +627,7 @@ export const useSearchDocs = (defaultLimit) => {
try {
const mimetype = docFilterToMimetype(filter);
let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject,
mimetype, deep);
mimetype, deep, anyTag);
setDocs(docs_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -636,7 +638,7 @@ export const useSearchDocs = (defaultLimit) => {
useEffect(() => {
localSearchDocs();
}, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject, filter, deep]);
}, [searchStr, limit, order, offset, JSON.stringify(activeTags), anyTag, author, subject, filter, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -686,7 +688,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
setLoading(true);
try {
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline,
censored, deep);
censored, deep, anyTag);
setVideos(videos_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -699,7 +701,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
useEffect(() => {
localSearchVideos();
}, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline, censored, deep]);
}, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), anyTag, headline, censored, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});

View file

@ -2,12 +2,12 @@ import React from 'react';
import {act, renderHook} from '@testing-library/react';
import {render, screen} from '../test-utils';
import {
useBrowseFiles, usePages, useDriveTemperature, useDriveHealth, useSearchChannels, useSearchRecentFiles,
useSettings, useStatistics, useStatus, useVideoExtras,
useBrowseFiles, usePages, useDriveTemperature, useDriveHealth, useSearchArchives, useSearchChannels,
useSearchDocs, useSearchRecentFiles, useSearchVideos, useSettings, useStatistics, useStatus, useVideoExtras,
} from './customHooks';
import {
ApiDownError, filesSearch, getFiles, getSettings, getStatistics, getStatus, getVideoComments, getVideoDescription,
searchChannels,
ApiDownError, fetchDomains, filesSearch, getFiles, getSettings, getStatistics, getStatus, getVideoComments,
getVideoDescription, searchArchives, searchChannels, searchDocs, searchVideos,
} from '../api';
import {getControllerStats} from '../api/controller';
import {QueryContext, StatusContext} from '../contexts/contexts';
@ -51,6 +51,10 @@ jest.mock('../api', () => ({
getSettings: jest.fn(),
filesSearch: jest.fn(),
getFiles: jest.fn(),
fetchDomains: jest.fn(),
searchArchives: jest.fn(),
searchDocs: jest.fn(),
searchVideos: jest.fn(),
}));
jest.mock('../api/controller', () => ({
@ -769,3 +773,35 @@ describe('useBrowseFiles', () => {
expect(result.current.browseFiles).toEqual([{path: 'renamed/'}]);
});
});
describe('module searches send the Any Tag filter', () => {
// `anyTag` is the last argument of each search helper.
const cases = [
['useSearchVideos', () => useSearchVideos(), searchVideos],
['useSearchArchives', () => useSearchArchives(), searchArchives],
['useSearchDocs', () => useSearchDocs(), searchDocs],
];
beforeEach(() => {
fetchDomains.mockResolvedValue([[], 0]);
searchVideos.mockResolvedValue([[], 0]);
searchArchives.mockResolvedValue([[], 0]);
searchDocs.mockResolvedValue([[], 0]);
});
afterEach(() => jest.clearAllMocks());
it.each(cases)('%s passes anyTag=true from the URL', async (_, hook, search) => {
renderHook(hook, {wrapper: makeWrapper({anyTag: 'true'})});
await act(async () => {});
expect(search).toHaveBeenCalled();
expect(search.mock.calls.at(-1).at(-1)).toBe(true);
});
it.each(cases)('%s passes anyTag=false without it', async (_, hook, search) => {
renderHook(hook, {wrapper: makeWrapper()});
await act(async () => {});
expect(search).toHaveBeenCalled();
expect(search.mock.calls.at(-1).at(-1)).toBe(false);
});
});

View file

@ -67,7 +67,7 @@ async def search_archives(request: Request, body: schema.ArchiveSearchRequest):
file_groups, total = await asyncio.to_thread(
lib.search_archives, request.ctx.session, search_str, domain, limit, offset, body.order_by, body.tag_names,
body.headline, body.deep)
body.headline, body.deep, body.any_tag)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)

View file

@ -1310,7 +1310,7 @@ NO_NULL_ORDERS = {
def search_archives(session: Session, search_str: str, domain: str, limit: int, offset: int, order: str,
tag_names: List[str], headline: bool = False, deep: bool = False) \
tag_names: List[str], headline: bool = False, deep: bool = False, any_tag: bool = False) \
-> Tuple[List[dict], int]:
# Always filter FileGroups to Archives.
wheres = []
@ -1360,7 +1360,7 @@ def search_archives(session: Session, search_str: str, domain: str, limit: int,
else ARCHIVE_ORDERS['published_datetime']
tag_names = tag_names or []
wheres, params = tag_append_sub_select_where(wheres, params, tag_names)
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
if domain:
params['domain'] = domain
@ -1394,7 +1394,7 @@ def search_archives(session: Session, search_str: str, domain: str, limit: int,
'''.strip()
logger.debug(f'{stmt} {params}')
unfiltered = not search_str and not tag_names and not domain and not null_filter
unfiltered = not search_str and not tag_names and not any_tag and not domain and not null_filter
if unfiltered:
total = cached_search_total(
search_filter_cache_key('archives'),
@ -1417,7 +1417,7 @@ def search_archives(session: Session, search_str: str, domain: str, limit: int,
LEFT JOIN file_group fg ON fg.id = a.file_group_id'''
count_stmt = f'SELECT COUNT(*) AS total {count_from} {where}'
cache_key = search_filter_cache_key(
'archives', search_str=search_str, domain=domain, tag_names=tag_names,
'archives', search_str=search_str, domain=domain, tag_names=tag_names, any_tag=any_tag,
deep=deep, null_filter=null_filter,
)
total = cached_search_total(cache_key, lambda: count_file_groups(session, count_stmt, params))

View file

@ -1,6 +1,8 @@
from dataclasses import dataclass, field
from typing import List, Optional
from wrolpi.errors import ValidationError
@dataclass
class ArchiveDict:
@ -25,6 +27,11 @@ class ArchiveSearchRequest:
tag_names: List[str] = field(default_factory=list)
headline: bool = False
deep: bool = False # When True, search all text including page contents (d_text)
any_tag: bool = False # When True, only Archives with at least one Tag.
def __post_init__(self):
if self.any_tag and self.tag_names:
raise ValidationError('Cannot use both tag_names and any_tag.')
@dataclass

View file

@ -119,6 +119,31 @@ async def test_search_archive_tags(test_session, async_client, archive_factory,
assert response.json['totals']['file_groups'] == 0
@pytest.mark.asyncio
async def test_search_archive_any_tag(test_session, async_client, archive_factory, tag_factory):
"""Archives can be filtered to those with any Tag."""
tag1, tag2 = await tag_factory(), await tag_factory()
one = archive_factory(domain='example.com', tag_names=[tag1.name, ])
two = archive_factory(domain='example.org', tag_names=[tag2.name, ])
untagged = archive_factory(domain='example.com')
test_session.commit()
async def search_ids(**body):
request, response = await async_client.post('/api/archive/search', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK, response.json
assert response.json['totals']['file_groups'] == len(response.json['file_groups'])
return {i['id'] for i in response.json['file_groups']}
assert await search_ids(any_tag=True) == {one.file_group_id, two.file_group_id}
assert await search_ids(any_tag=True, domain='example.com') == {one.file_group_id}
assert await search_ids() == {one.file_group_id, two.file_group_id, untagged.file_group_id}
# Cannot search for any_tag, and tag names.
content = dict(tag_names=[tag1.name], any_tag=True)
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_archives_search_headline(test_session, archive_directory, archive_factory, async_client):
"""Headlines can be requested."""

View file

@ -66,6 +66,7 @@ async def search_docs(request: Request, body: schema.DocSearchRequest):
order_by=body.order_by,
tag_names=body.tag_names,
deep=body.deep,
any_tag=body.any_tag,
)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)

View file

@ -13,8 +13,9 @@ from wrolpi.collections.types import collection_type_registry
from wrolpi.errors import ValidationError
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype, cached_search_total, \
search_filter_cache_key
from wrolpi.tags import TagFile, tag_names_to_file_group_sub_select
logger = logging.getLogger(__name__)
logger =logging.getLogger(__name__)
# Register author and subject collection types.
collection_type_registry.register(
@ -371,7 +372,7 @@ def _doc_browse_sql(order_by: str) -> Optional[str]:
def _search_docs(session: Session, search_str=None, author=None, subject=None, language=None, mimetype=None,
limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False):
limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False, any_tag=False):
from .models import Doc
from wrolpi.files.models import FileGroup
@ -402,14 +403,18 @@ def _search_docs(session: Session, search_str=None, author=None, subject=None, l
if mimetype:
query = query.filter(FileGroup.mimetype.startswith(mimetype))
if any_tag and tag_names:
raise ValidationError('Cannot use both tag_names and any_tag.')
if tag_names:
from wrolpi.tags import Tag, TagFile
tagged_fg_ids = session.query(TagFile.file_group_id) \
.join(Tag, Tag.id == TagFile.tag_id) \
.filter(Tag.name.in_(tag_names))
# A Doc must have every Tag, like the other searches.
tags_sub_select, tags_params = tag_names_to_file_group_sub_select(tag_names, dict())
tagged_fg_ids = text(tags_sub_select).bindparams(**tags_params) \
.columns(column('file_group_id', Integer))
query = query.filter(FileGroup.id.in_(tagged_fg_ids))
elif any_tag:
query = query.filter(FileGroup.id.in_(session.query(TagFile.file_group_id)))
unfiltered = not (search_str or author or subject or language or mimetype or tag_names)
unfiltered = not (search_str or author or subject or language or mimetype or tag_names or any_tag)
if unfiltered:
total = cached_search_total(
search_filter_cache_key('docs'),
@ -418,7 +423,7 @@ def _search_docs(session: Session, search_str=None, author=None, subject=None, l
else:
cache_key = search_filter_cache_key(
'docs', search_str=search_str, author=author, subject=subject,
language=language, mimetype=mimetype, tag_names=tag_names, deep=deep,
language=language, mimetype=mimetype, tag_names=tag_names, deep=deep, any_tag=any_tag,
)
# Count before ORDER BY / LIMIT; cache by filters not page.
total = cached_search_total(cache_key, lambda q=query: q.count())

View file

@ -1,6 +1,8 @@
import dataclasses
from typing import Optional
from wrolpi.errors import ValidationError
@dataclasses.dataclass
class DocSearchRequest:
@ -14,6 +16,11 @@ class DocSearchRequest:
limit: int = 20
offset: int = 0
deep: bool = False # When True, search all text including document contents (d_text)
any_tag: bool = False # When True, only Docs with at least one Tag.
def __post_init__(self):
if self.any_tag and self.tag_names:
raise ValidationError('Cannot use both tag_names and any_tag.')
@dataclasses.dataclass

View file

@ -98,6 +98,36 @@ async def test_docs_search_by_tag(async_client, test_session, doc_factory, tag_f
assert response.json['totals']['file_groups'] == 0
@pytest.mark.asyncio
async def test_docs_search_tag_names_and_any_tag(async_client, test_session, doc_factory, tag_factory):
"""Docs tag filtering matches the other searches: `tag_names` requires every tag, `any_tag` requires some tag."""
tag1, tag2 = await tag_factory(), await tag_factory()
one_tag, both_tags, untagged = doc_factory(), doc_factory(), doc_factory()
one_tag.file_group.add_tag(test_session, tag1.id)
both_tags.file_group.add_tag(test_session, tag1.id)
both_tags.file_group.add_tag(test_session, tag2.id)
test_session.commit()
async def search_ids(**body):
request, response = await async_client.post('/api/docs/search', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK, response.json
assert response.json['totals']['file_groups'] == len(response.json['file_groups'])
return {i['id'] for i in response.json['file_groups']}
assert await search_ids(tag_names=[tag1.name]) == {one_tag.file_group_id, both_tags.file_group_id}
# Only one Doc has both Tags.
assert await search_ids(tag_names=[tag1.name, tag2.name]) == {both_tags.file_group_id}
# Any Tag excludes only the untagged Doc.
assert await search_ids(any_tag=True) == {one_tag.file_group_id, both_tags.file_group_id}
assert await search_ids(any_tag=False) == {one_tag.file_group_id, both_tags.file_group_id,
untagged.file_group_id}
# Cannot search for any_tag, and tag names.
content = dict(tag_names=[tag1.name], any_tag=True)
request, response = await async_client.post('/api/docs/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_delete_doc(async_client, test_session, test_directory, example_epub, refresh_files):
"""Docs can be deleted, including their on-disk files and FileGroup row."""

View file

@ -337,6 +337,7 @@ def assert_video_search(async_client):
channel_id: int = None,
censored: bool = None,
deep: bool = None,
any_tag: bool = None,
):
content = dict()
if search_str is not None:
@ -355,6 +356,8 @@ def assert_video_search(async_client):
content['censored'] = censored
if deep is not None:
content['deep'] = deep
if any_tag is not None:
content['any_tag'] = any_tag
request, response = await async_client.post('/api/videos/search', content=json.dumps(content))

View file

@ -155,6 +155,11 @@ class VideoSearchRequest:
headline: bool = False
censored: bool = False
deep: bool = False # When True, search all text including captions (d_text)
any_tag: bool = False # When True, only Videos with at least one Tag.
def __post_init__(self):
if self.any_tag and self.tag_names:
raise ValidationError('Cannot use both tag_names and any_tag.')
@dataclass

View file

@ -270,6 +270,32 @@ async def test_search_videos(test_session, video_factory, assert_video_search, s
# No results, no total is returned from the DB.
await assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2)
@pytest.mark.asyncio
async def test_search_videos_any_tag(test_session, async_client, video_factory, assert_video_search, tag_factory):
"""Videos can be filtered to those with any Tag, in the default date order and in every other order."""
vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2')
vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
tag1, tag2 = await tag_factory(), await tag_factory()
vid1.add_tag(test_session, tag1.id)
vid3.add_tag(test_session, tag2.id)
test_session.commit()
# The untagged vid2 is excluded; date orders take the unfiltered browse path without `any_tag`.
await assert_video_search(assert_total=2, assert_ids=[vid3.id, vid1.id], order_by='-published_datetime',
any_tag=True)
await assert_video_search(assert_total=2, assert_ids=[vid1.id, vid3.id], order_by='published_datetime',
any_tag=True)
await assert_video_search(assert_total=1, assert_ids=[vid3.id], search_str='vid3', any_tag=True)
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime',
any_tag=False)
# Cannot search for any_tag, and tag names.
content = dict(tag_names=[tag1.name], any_tag=True)
request, response = await async_client.post('/api/videos/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
# Check all order_by.
for order_by in video_lib.VIDEO_ORDERS:
await assert_video_search(order_by=order_by)

View file

@ -137,6 +137,7 @@ async def search_videos(request: Request, body: schema.VideoSearchRequest):
body.headline,
body.censored,
body.deep,
body.any_tag,
)
ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}}

View file

@ -197,6 +197,7 @@ def search_videos(
headline: bool = False,
censored: bool = False,
deep: bool = False,
any_tag: bool = False,
) -> Tuple[List[dict], int]:
tag_names = tag_names or []
# Search videos and audio-only files.
@ -210,7 +211,8 @@ def search_videos(
# Explicit JSON nulls bypass the schema defaults; SQLite rejects LIMIT/OFFSET NULL.
limit = int(limit) if limit else 20
offset = int(offset) if offset else 0
if not search_str and not tag_names and not channel_id and not censored and order in INDEXED_DATE_ORDERS:
if not search_str and not tag_names and not any_tag and not channel_id and not censored \
and order in INDEXED_DATE_ORDERS:
# The Videos page with no filters, in date order.
return _search_videos_by_date(session, order, limit, offset)
@ -232,7 +234,7 @@ def search_videos(
else:
select_columns = 'fg.id'
wheres, params = tag_append_sub_select_where(wheres, params, tag_names)
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
if fts_search and headline:
# The title headline is computed in Python by `handle_file_group_search_results`.
@ -277,7 +279,7 @@ def search_videos(
'''.strip()
logger.debug(f'{stmt} {params}')
unfiltered = (not search_str and not tag_names and not channel_id and not censored
unfiltered = (not search_str and not tag_names and not any_tag and not channel_id and not censored
and not null_filter)
if unfiltered:
total = cached_search_total(search_filter_cache_key('videos'), lambda: _count_videos(session))
@ -300,7 +302,7 @@ def search_videos(
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {join} {where}'
count_params = params
cache_key = search_filter_cache_key(
'videos', search_str=search_str, channel_id=channel_id, tag_names=tag_names,
'videos', search_str=search_str, channel_id=channel_id, tag_names=tag_names, any_tag=any_tag,
censored=censored, deep=deep, null_filter=null_filter,
)
total = cached_search_total(cache_key, lambda: count_file_groups(session, count_stmt, count_params))