From 3d6ccad0c05d937e372a7cced79a34f3c417b4d9 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 2 Feb 2025 10:16:11 -0700 Subject: [PATCH] Adding feature which adds toggles to Zim files to enable/disable auto search. --- .../versions/dc6637ced17b_zim_auto_search.py | 33 +++++++++++++++ app/src/DashboardPage.js | 1 + app/src/api.js | 8 ++++ app/src/components/Common.js | 11 +++-- app/src/components/Zim.js | 34 ++++++++++++---- modules/archive/api.py | 25 ++++++++---- modules/zim/api.py | 12 ++++++ modules/zim/lib.py | 7 ++++ modules/zim/models.py | 40 +++++++++++++++++-- modules/zim/schema.py | 5 +++ modules/zim/test/test_api.py | 37 +++++++++++++++++ 11 files changed, 192 insertions(+), 21 deletions(-) create mode 100644 alembic/versions/dc6637ced17b_zim_auto_search.py diff --git a/alembic/versions/dc6637ced17b_zim_auto_search.py b/alembic/versions/dc6637ced17b_zim_auto_search.py new file mode 100644 index 00000000..b97da0fa --- /dev/null +++ b/alembic/versions/dc6637ced17b_zim_auto_search.py @@ -0,0 +1,33 @@ +"""Zim auto search + +Revision ID: dc6637ced17b +Revises: 1f2325523525 +Create Date: 2025-01-30 19:02:32.692428 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'dc6637ced17b' +down_revision = '1f2325523525' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE zim ADD COLUMN auto_search BOOLEAN DEFAULT TRUE') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE zim DROP COLUMN IF EXISTS auto_search') diff --git a/app/src/DashboardPage.js b/app/src/DashboardPage.js index d2f4ed64..8053bd28 100644 --- a/app/src/DashboardPage.js +++ b/app/src/DashboardPage.js @@ -113,6 +113,7 @@ function EvaluateForm() {
sqrt(15)
15 km to miles
65 degF to degC
+
7 * 750 Mb
const inputRef = React.useRef(); diff --git a/app/src/api.js b/app/src/api.js index 8c833a39..33b1bf84 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1333,6 +1333,14 @@ export async function zimUnsubscribe(id) { return response.status === 204; } +export async function setZimAutoSearch(id, auto_search) { + const body = {auto_search}; + const response = await apiPost(`${API_URI}/zim/${id}/auto_search`, body); + if (response.status !== 204) { + throw new Error('Failed to set Zim auto search'); + } +} + export async function fetchDecoded(vinNumber) { const body = {vin_number: vinNumber}; const response = await apiPost(`${API_URI}/vin_number_decoder`, body); diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 5e10e8e3..c5ee41a1 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -795,7 +795,7 @@ export function ThrottleToggle() { ; } -export function Toggle({label, checked, disabled, onChange, icon}) { +export function Toggle({label, checked, disabled, onChange, icon, popupContent = null}) { // Custom toggle because Semantic UI does not handle inverted labels correctly. const {t, inverted} = useContext(ThemeContext); @@ -833,7 +833,7 @@ export function Toggle({label, checked, disabled, onChange, icon}) { icon = } - return <> + const body =
@@ -843,7 +843,12 @@ export function Toggle({label, checked, disabled, onChange, icon}) { {icon} {label} - + + + if (popupContent) { + return + } + return body } export function DisableDownloadsToggle() { diff --git a/app/src/components/Zim.js b/app/src/components/Zim.js index 30cb3e77..15bcfc11 100644 --- a/app/src/components/Zim.js +++ b/app/src/components/Zim.js @@ -42,6 +42,7 @@ import { Paginator, TabLinks, TagIcon, + Toggle, useTitle, WarningMessage } from "./Common"; @@ -53,6 +54,7 @@ import { fetchZimSubscriptions, refreshFiles, saveSettings, + setZimAutoSearch, tagZimEntry, untagZimEntry, zimSubscribe, @@ -343,9 +345,9 @@ const ZimCatalogItemRow = ({item, subscriptions, iso_639_codes, fetchSubscriptio const subscription = name in subscriptions ? subscriptions[name] : null; const subscriptionLanguage = subscription ? subscription['language'] : 'en'; - const [langauge, setLanguage] = useState(subscriptionLanguage); + const [language, setLanguage] = useState(subscriptionLanguage); const [pending, setPending] = useState(false); - const languageChange = subscription ? langauge !== subscription['language'] : false; + const languageChange = subscription ? language !== subscription['language'] : false; const wrolModeEnabled = useWROLMode(); @@ -356,7 +358,7 @@ const ZimCatalogItemRow = ({item, subscriptions, iso_639_codes, fetchSubscriptio if (subscription && !languageChange) { success = await zimUnsubscribe(subscription['id']); } else { - success = await zimSubscribe(name, langauge); + success = await zimSubscribe(name, language); } } catch (e) { console.error(e); @@ -387,7 +389,7 @@ const ZimCatalogItemRow = ({item, subscriptions, iso_639_codes, fetchSubscriptio const languageDropdown = ; @@ -445,12 +447,28 @@ class ManageZim extends React.Component { } } - zimFileTableRow = (zim, sortData) => { - const {path, size} = zim; + zimFileTableRow = (zim, sortData, localFetchZims) => { + const {id, path, size, auto_search} = zim; + + const toggleZimAutoSearch = async () => { + try { + await setZimAutoSearch(id, !auto_search); + } catch (e) { + throw e; + } finally { + await localFetchZims(); + } + } + const toggle = return {path} {humanFileSize(size)} + {toggle} } @@ -460,6 +478,7 @@ class ManageZim extends React.Component { const zimFilesHeaders = [ {key: 'path', text: 'Path', sortBy: 'path', width: 14}, {key: 'size', text: 'Size', sortBy: 'size', width: 2}, + {key: 'search', text: 'Search', sortBy: 'auto_search', width: 2}, ]; let zimFilesBody = @@ -469,8 +488,9 @@ class ManageZim extends React.Component { ; if (zims && zims.length >= 1) { zimFilesBody = this.zimFileTableRow(i, sortData, this.fetchZims.bind(this))} rowKey='path' tableHeaders={zimFilesHeaders} />; diff --git a/modules/archive/api.py b/modules/archive/api.py index 8dfbb4e5..240900ce 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -74,14 +74,6 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest): return json_response(ret) -@register_switch_handler('singlefile_upload_switch_handler') -async def singlefile_upload_switch_handler(singlefile): - archive = await lib.singlefile_to_archive(singlefile) - logger.info(f'Created Archive from upload: {archive}') - name = archive.file_group.title or archive.file_group.url - Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location) - - @archive_bp.get('/upload') @openapi.definition( summary='A message to confirm to the user that they have the correct upload URL.' @@ -91,6 +83,23 @@ async def get_upload_singlefile(request: Request): ' This requires a POST request.') +@register_switch_handler('singlefile_upload_switch_handler') +async def singlefile_upload_switch_handler(singlefile): + """Used by `post_upload_singlefile` to upload a single file""" + from wrolpi.downloader import download_manager, Download + from . import ArchiveDownloader + + archive = await lib.singlefile_to_archive(singlefile) + logger.info(f'Created Archive from upload: {archive}') + name = archive.file_group.title or archive.file_group.url + Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location) + url = archive.file_group.url + if url and (download := Download.get_by_url(url)): + if download.is_failed and download.downloader == ArchiveDownloader.name: + # Download was attempted and failed, user manually archived the URL. + download_manager.delete_download(download.id) + + @archive_bp.post('/upload') @openapi.definition( summary='Upload SingleFile from SingleFile browser extension and convert it to an Archive.' diff --git a/modules/zim/api.py b/modules/zim/api.py index f5a526fc..7ba16fb5 100644 --- a/modules/zim/api.py +++ b/modules/zim/api.py @@ -43,6 +43,18 @@ async def delete_zims(_: Request, zim_ids: str): return response.empty() +@zim_bp.post('//auto_search') +@openapi.definition( + summary='Change if a Zim file will be searched by default.', + body=schema.ZimAutoSearchRequest, +) +@validate(schema.ZimAutoSearchRequest) +async def post_set_zim_auto_search(_: Request, zim_id: int, body: schema.ZimAutoSearchRequest): + zim_id = int(zim_id) + lib.set_zim_auto_search(zim_id, body.auto_search) + return response.empty() + + @zim_bp.post('/search/') @openapi.definition( summary='Search all entries of a Zim', diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 076971e8..7f17828d 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -572,3 +572,10 @@ async def restart_kiwix(): if result.return_code != 0 and result.stderr: logger.debug(result.stderr) return result.return_code + + +@optional_session +def set_zim_auto_search(zim_id: int, auto_search: bool, session: Session = None): + zim = Zim.find_by_id(zim_id, session) + zim.auto_search = auto_search + session.commit() diff --git a/modules/zim/models.py b/modules/zim/models.py index 7441977a..bc9e6c8a 100644 --- a/modules/zim/models.py +++ b/modules/zim/models.py @@ -6,11 +6,11 @@ from datetime import datetime from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional, Set from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher -from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_ +from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean from sqlalchemy.orm import relationship, Session from sqlalchemy.orm.exc import NoResultFound # noqa -from modules.zim.errors import UnknownZimEntry, UnknownZimTagEntry +from modules.zim.errors import UnknownZimEntry, UnknownZimTagEntry, UnknownZim from wrolpi import dates, tags from wrolpi.common import Base, logger, get_relative_to_media_directory from wrolpi.dates import TZDateTime @@ -72,6 +72,7 @@ class Zim(Base): file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False) file_group: FileGroup = relationship('FileGroup') + auto_search = Column(Boolean, nullable=False, default=True) def __repr__(self): return f'' @@ -83,6 +84,7 @@ class Zim(Base): file_group_id=self.file_group_id, metadata=self.zim_metadata, size=self.file_group.size, + auto_search=self.auto_search, ) return d @@ -94,6 +96,37 @@ class Zim(Base): session.flush([file_group, zim]) return zim + @staticmethod + @optional_session + def get_by_path(path: pathlib.Path, session: Session = None) -> Optional['Zim']: + zim = session.query(Zim).filter_by(path=path).one_or_none() + if not zim and path.is_file(): + zim = Zim.from_paths(session, path) + session.commit() + return zim + + @staticmethod + @optional_session + def find_by_path(path: pathlib.Path, session: Session = None) -> 'Zim': + zim = Zim.get_by_path(path, session=session) + if not zim: + raise UnknownZim(f'Cannot find zim at path') + return zim + + @staticmethod + @optional_session + def get_by_id(zim_id: int, session: Session = None) -> Optional['Zim']: + zim = session.query(Zim).filter_by(id=zim_id).one_or_none() + return zim + + @staticmethod + @optional_session + def find_by_id(zim_id: int, session: Session = None) -> 'Zim': + zim = Zim.get_by_id(zim_id, session=session) + if not zim: + raise UnknownZim(f'Cannot find zim with ID {zim_id}') + return zim + def get_zim(self) -> Archive: if not self.path or not self.path.is_file(): raise FileNotFoundError(f'{self} does not exist!') @@ -295,7 +328,8 @@ class Zims: zims = cls.get_all(session=session) results = OrderedDict() for zim in zims: - results[zim] = zim.estimate(search_str) + if zim.auto_search: + results[zim] = zim.estimate(search_str) return results @classmethod diff --git a/modules/zim/schema.py b/modules/zim/schema.py index d6042d2a..f4a8d01e 100644 --- a/modules/zim/schema.py +++ b/modules/zim/schema.py @@ -10,6 +10,11 @@ class ZimSearchRequest: limit: Optional[int] = 10 +@dataclass +class ZimAutoSearchRequest: + auto_search: bool + + @dataclass class ZimMetadata: path: str diff --git a/modules/zim/test/test_api.py b/modules/zim/test/test_api.py index 084092cc..c449dcc9 100644 --- a/modules/zim/test/test_api.py +++ b/modules/zim/test/test_api.py @@ -241,3 +241,40 @@ async def test_search_estimates(test_session, async_client, zim_factory): assert response.status_code == HTTPStatus.OK assert len(response.json['zims_estimates']) == 1 assert response.json['zims_estimates'][0]['estimate'] == 2 + + +@pytest.mark.asyncio +async def test_zim_auto_search(test_session, async_client, zim_factory): + """The auto-searching of any Zim file can be toggled.""" + zim = zim_factory('test_zim_auto_search.zim') + assert zim.auto_search is True, 'Zim should be searched by default.' + + # Zim is estimated by default. + body = dict(search_str='one') + request, response = await async_client.post('/api/zim/search_estimates', json=body) + assert response.status_code == HTTPStatus.OK + assert len(response.json['zims_estimates']) == 1 + + # Change auto_search so Zim is not estimated by default. + body = dict(auto_search=False) + request, response = await async_client.post(f'/api/zim/{zim.id}/auto_search', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT + assert zim.auto_search is False + + # Estimates are empty because the only Zim was filtered out. + body = dict(search_str='one') + request, response = await async_client.post('/api/zim/search_estimates', json=body) + assert response.status_code == HTTPStatus.OK + assert len(response.json['zims_estimates']) == 0 + + # Change auto_search so Zim is now estimated again. + body = dict(auto_search=True) + request, response = await async_client.post(f'/api/zim/{zim.id}/auto_search', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT + assert zim.auto_search is True + + # Zim is estimated by default. + body = dict(search_str='one') + request, response = await async_client.post('/api/zim/search_estimates', json=body) + assert response.status_code == HTTPStatus.OK + assert len(response.json['zims_estimates']) == 1