Add Domains to the Other search tab

Domains match the search text by name and/or any of the Tags, like Channels.
The list and its estimate share one query.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/src/api.js

@@ -2129,6 +2129,16 @@ export async function searchChannels(tagNames, searchStr) {
     }
 }
 
+// Domains whose name matches `searchStr` and/or which have any of the tags.
+export async function searchDomains(tagNames, searchStr) {
+    const body = {tag_names: tagNames, search_str: searchStr};
+    const response = await apiPost(`${ARCHIVES_API}/domains/search`, body);
+    if (response.ok) {
+        const content = await response.json();
+        return {domains: content.domains};
+    }
+}
+
 export async function getOutdatedZims() {
     const response = await apiGet(`${API_URI}/zim/outdated`);
     if (response.status === 200) {

app/src/components/OtherSearchView.test.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import {act, render, screen} from '../test-utils';
 import {OtherSearchView} from './Search';
-import {searchChannels, searchRepos} from '../api';
+import {searchChannels, searchDomains, searchRepos} from '../api';
 
 // What the search page's Other tab shows for each state of useSearchChannels.  "No Channels"
 // must only appear once the server has confirmed there are none.
@@ -9,6 +9,7 @@ import {searchChannels, searchRepos} from '../api';
 jest.mock('../api', () => ({
     ...jest.requireActual('../api'),
     searchChannels: jest.fn(),
+    searchDomains: jest.fn(),
     searchRepos: jest.fn(),
 }));
 
@@ -18,6 +19,8 @@ beforeEach(() => {
     searchChannels.mockReset();
     searchRepos.mockReset();
     searchRepos.mockResolvedValue({repos: [], total: 0});
+    searchDomains.mockReset();
+    searchDomains.mockResolvedValue({domains: []});
 });
 
 test('pending: shows a loader, not "No Channels"', () => {
@@ -117,3 +120,40 @@ describe('Repos', () => {
         expect(screen.getByText(/files <script>/)).toBeInTheDocument();
     });
 });
+
+describe('Domains', () => {
+    test('empty: shows "No Domains"', async () => {
+        searchChannels.mockResolvedValue({channels: []});
+        render(<OtherSearchView loading={false}/>);
+        await act(async () => {
+        });
+        expect(screen.getByText('No Domains')).toBeInTheDocument();
+    });
+
+    test('failed: shows an error only in the Domains section', async () => {
+        searchChannels.mockResolvedValue({channels: [{id: 1, name: 'Cooking'}]});
+        searchDomains.mockRejectedValue(new Error('boom'));
+        const spy = jest.spyOn(console, 'error').mockImplementation(() => {
+        });
+        try {
+            render(<OtherSearchView loading={false}/>);
+            await act(async () => {
+            });
+            expect(screen.getByText('Cooking')).toBeInTheDocument();
+            expect(screen.getByText(/Could not fetch the domains/)).toBeInTheDocument();
+        } finally {
+            spy.mockRestore();
+        }
+    });
+
+    test('loaded: lists domains by the search, linked to their archives', async () => {
+        searchChannels.mockResolvedValue({channels: []});
+        searchDomains.mockResolvedValue({domains: [{id: 3, domain: 'example.com', tag_name: null}]});
+        const searchParams = new URLSearchParams('q=example&tag=News');
+        render(<OtherSearchView loading={false}/>, {query: {searchParams}});
+        await act(async () => {
+        });
+        expect(searchDomains).toHaveBeenCalledWith(['News'], 'example');
+        expect(screen.getByText('example.com').closest('a')).toHaveAttribute('href', '/archives?domain=example.com');
+    });
+});

app/src/components/Search.js

@@ -8,6 +8,7 @@ import {
     useSearchDate,
     useSearchDeep,
     useSearchFilter,
+    useSearchDomains,
     useSearchRepos,
 } from "../hooks/customHooks";
 import {ShortcutHint} from "./ShortcutHint";
@@ -562,6 +563,16 @@ function SearchChannelPreview({channel}) {
     </div>
 }
 
+function SearchDomainPreview({domain}) {
+    const {SingleTag} = React.useContext(TagsContext);
+    return <div>
+        <span style={{display: 'inline-flex', alignItems: 'center', gap: 8, flexWrap: 'wrap'}}>
+            <Link to={`/archives?domain=${domain.domain}`}>{domain.domain}</Link>
+            {domain.tag_name && <SingleTag name={domain.tag_name}/>}
+        </span>
+    </div>
+}
+
 function SearchRepoPreview({repo}) {
     const {SingleTag} = React.useContext(TagsContext);
     return <div>
@@ -594,14 +605,15 @@ function OtherSearchSection({value, title, items, preview, emptyText, errorText}
     </Accordion.Item>
 }
 
-// `channels` follows useSearchChannels and `repos` follows useSearchRepos: null = pending,
+// `channels`, `domains`, and `repos` follow useSearchChannels, useSearchDomains, and useSearchRepos: null = pending,
 // undefined = fetch failed, [] = none.
 export function OtherSearchView({loading}) {
     const {searchParams} = React.useContext(QueryContext);
-    const [activeValues, setActiveValues] = React.useState(['channels', 'repos']);
+    const [activeValues, setActiveValues] = React.useState(['channels', 'domains', 'repos']);
     const activeTags = searchParams.getAll('tag');
     const searchStr = searchParams.get('q') || '';
     const {channels, loading: channelsLoading} = useSearchChannels(activeTags, searchStr);
+    const {domains} = useSearchDomains(searchStr, activeTags);
     const {repos} = useSearchRepos(searchStr, activeTags);
 
     if (loading || channelsLoading || channels === null) {
@@ -619,6 +631,14 @@ export function OtherSearchView({loading}) {
             emptyText='No Channels'
             errorText='Could not fetch the channels.'
         />
+        <OtherSearchSection
+            value='domains'
+            title='Domains'
+            items={domains}
+            preview={i => <SearchDomainPreview key={i.id} domain={i}/>}
+            emptyText='No Domains'
+            errorText='Could not fetch the domains.'
+        />
         <OtherSearchSection
             value='repos'
             title='Repos'

app/src/hooks/customHooks.js

@@ -36,6 +36,7 @@ import {
     saveSettings,
     searchArchives,
     searchChannels,
+    searchDomains,
     searchDirectories,
     searchDocs,
     searchRepos,
@@ -2158,6 +2159,27 @@ export const useSearchChannels = (defaultTagNames, searchStr) => {
     }
 }
 
+// Domains whose name matches `searchStr` and/or which have any of the tags.  `domains`: null = pending,
+// undefined = fetch failed, [] = none.  Only the newest request may write state.
+export const useSearchDomains = (searchStr, tagNames) => {
+    const [domains, setDomains] = useState(null);
+    const requestGen = useRef(0);
+    const tagKey = JSON.stringify(tagNames || []);
+
+    useEffect(() => {
+        const gen = ++requestGen.current;
+        setDomains(null);
+        searchDomains(JSON.parse(tagKey), searchStr)
+            .then(result => gen === requestGen.current && setDomains(result ? result.domains : undefined))
+            .catch(e => {
+                console.error(e);
+                if (gen === requestGen.current) setDomains(undefined);
+            });
+    }, [searchStr, tagKey]);
+
+    return {domains};
+}
+
 // Repos matching the search (their name and README) and/or the tags.  `repos`: null = pending,
 // undefined = fetch failed, [] = none.  Only the newest request may write state.
 export const useSearchRepos = (searchStr, tagNames) => {

modules/archive/api.py

@@ -68,6 +68,17 @@ async def search_archives(request: Request, body: schema.ArchiveSearchRequest):
     return json_response(ret)
 
 
+@archive_bp.post('/domains/search')
+@openapi.definition(
+    description='Search Domains by their name and/or Tags',
+    body=schema.DomainSearchRequest,
+)
+@validate(schema.DomainSearchRequest)
+async def domain_search(request: Request, body: schema.DomainSearchRequest):
+    domains = lib.search_domains(request.ctx.session, body.tag_names, body.search_str)
+    return json_response(dict(domains=domains))
+
+
 @archive_bp.post('/file_format')
 @openapi.definition(
     description='Preview the archive file format',

modules/archive/lib.py

@@ -16,7 +16,7 @@ from urllib.parse import urlparse
 
 import pytz
 from sqlalchemy import asc, func, BigInteger
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Query, Session
 
 from modules.archive.models import Archive
 from wrolpi import dates
@@ -34,7 +34,7 @@ from wrolpi.files.lib import handle_file_group_search_results, cached_search_tot
     search_filter_cache_key, count_file_groups
 from wrolpi.files.models import FileGroup
 from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
-from wrolpi.tags import tag_append_sub_select_where
+from wrolpi.tags import Tag, tag_append_sub_select_where
 from wrolpi.vars import PYTEST, DOCKERIZED
 
 logger = logger.getChild(__name__)
@@ -1555,6 +1555,29 @@ async def search_domains_by_name(session: Session, name: str, limit: int = 5) ->
     ]
 
 
+def search_domains_query(session: Session, tag_names: List[str] = None, search_str: str = None) -> Optional[Query]:
+    """Domains whose name contains `search_str` and which have any of the Tags.  None when there is nothing to search
+    by."""
+    search_str = (search_str or '').strip()
+    if not tag_names and not search_str:
+        return None
+    query = session.query(Collection).filter(Collection.kind == 'domain')
+    if search_str:
+        query = query.filter(Collection.name.ilike(f'%{search_str}%'))
+    if tag_names:
+        query = query.join(Tag, Tag.id == Collection.tag_id).filter(Tag.name.in_(tag_names))
+    return query
+
+
+def search_domains(session: Session, tag_names: List[str] = None, search_str: str = None) -> List[dict]:
+    """Search Domains by their name and/or Tags."""
+    query = search_domains_query(session, tag_names, search_str)
+    if query is None:
+        return []
+    collections = query.order_by(func.lower(Collection.name)).all()
+    return [dict(id=i.id, domain=i.name, tag_name=i.tag_name) for i in collections]
+
+
 async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
     """Extract the readability dict from the provided HTML."""
     with tempfile.NamedTemporaryFile('wb', suffix='.html') as singlefile_file:

modules/archive/schema.py

@@ -27,6 +27,12 @@ class ArchiveSearchRequest:
     deep: bool = False  # When True, search all text including page contents (d_text)
 
 
+@dataclass
+class DomainSearchRequest:
+    tag_names: List[str] = field(default_factory=list)
+    search_str: Optional[str] = None
+
+
 @dataclass
 class ArchiveSearchResponse:
     file_groups: List[ArchiveDict]

modules/archive/test/test_domain_search.py

@@ -0,0 +1,46 @@
+from http import HTTPStatus
+
+import pytest
+
+from wrolpi.collections import Collection
+
+
+@pytest.mark.asyncio
+async def test_search_domains(async_client, test_session, tag_factory):
+    """Domains can be searched by their name and/or Tags, and the Other tab's estimate counts the same Domains."""
+    tag1, tag2 = await tag_factory(), await tag_factory()
+    test_session.add_all([
+        Collection(name='cooking.example.com', kind='domain', tag=tag1),
+        Collection(name='cooking.example.org', kind='domain'),
+        Collection(name='gardening.example.com', kind='domain', tag=tag2),
+        # Not a Domain.
+        Collection(name='cooking channel', kind='channel', tag=tag1),
+    ])
+    test_session.commit()
+
+    async def search(body):
+        request, response = await async_client.post('/api/archive/domains/search', json=body)
+        assert response.status_code == HTTPStatus.OK
+        return sorted(i['domain'] for i in response.json['domains'])
+
+    cases = [
+        (dict(search_str='cooking'), ['cooking.example.com', 'cooking.example.org']),
+        (dict(tag_names=[tag1.name]), ['cooking.example.com']),
+        # Any of the Tags.
+        (dict(tag_names=[tag1.name, tag2.name]), ['cooking.example.com', 'gardening.example.com']),
+        # The name and the Tags must both match.
+        (dict(search_str='example.com', tag_names=[tag2.name]), ['gardening.example.com']),
+        (dict(search_str='cooking', tag_names=[tag2.name]), []),
+        # Nothing to search by matches nothing.
+        (dict(search_str=''), []),
+        (dict(), []),
+    ]
+    for body, expected in cases:
+        assert await search(body) == expected, body
+        request, response = await async_client.post('/api/search_other_estimates', json=body)
+        assert response.status_code == HTTPStatus.OK
+        assert response.json['others']['domain_count'] == len(expected), body
+
+    request, response = await async_client.post('/api/archive/domains/search', json=dict(tag_names=[tag1.name]))
+    domain, = response.json['domains']
+    assert domain['tag_name'] == tag1.name

modules/repos/test/test_search.py

@@ -116,13 +116,13 @@ async def test_search_api(test_session, async_client, test_wrolpi_config):
 
     # The global search's "Other" estimate counts Repos by the search, and by Tag.
     request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
-    assert response.json['others'] == dict(channel_count=0, repo_count=1)
+    assert response.json['others'] == dict(channel_count=0, domain_count=0, repo_count=1)
     body = dict(tag_names=['software'])
     request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
-    assert response.json['others'] == dict(channel_count=0, repo_count=1)
+    assert response.json['others'] == dict(channel_count=0, domain_count=0, repo_count=1)
     body = dict(tag_names=['software'], search_str='nothing')
     request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
-    assert response.json['others'] == dict(channel_count=0, repo_count=0)
+    assert response.json['others'] == dict(channel_count=0, domain_count=0, repo_count=0)
 
     # Repos are suggested by name.
     request, response = await async_client.post('/api/search_suggestions', content=json.dumps(dict(search_str='kiw')))

wrolpi/common.py

@@ -2870,14 +2870,17 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
 
 
 async def search_other_estimates(session: Session, tag_names: List[str], search_str: str = None) -> dict:
-    """Estimate other things that match the search or the Tags (Channels by their name, Repos by their name and
-    README)."""
+    """Estimate other things that match the search or the Tags (Channels and Domains by their name, Repos by their
+    name and README)."""
+    from modules.archive.lib import search_domains_query
     from modules.videos.channel.lib import search_channels_query
     from modules.repos.lib import count_repos
 
-    query = search_channels_query(session, tag_names, search_str)
+    channels = search_channels_query(session, tag_names, search_str)
+    domains = search_domains_query(session, tag_names, search_str)
     others = dict(
-        channel_count=query.count() if query is not None else 0,
+        channel_count=channels.count() if channels is not None else 0,
+        domain_count=domains.count() if domains is not None else 0,
         repo_count=count_repos(session, search_str, tag_names),
     )
     return others