2023-05-29 18:04:03 -06:00
|
|
|
import datetime
|
|
|
|
|
from http import HTTPStatus
|
|
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
from libzim import Entry
|
|
|
|
|
|
Zim estimates and searches run in a libzim worker process so a cold Zim cannot stall the API worker (#928)
* Zim search estimates run in a separate process so a cold estimate cannot stall the API worker
The global search estimates every auto-search Zim on each keystroke. On a
large library that estimate took tens of seconds when the Zims' pages were not
cached, and every other request routed to the same Sanic worker, such as the
channel suggestions, waited behind it. Two things held the worker: opening a
large Zim cold takes seconds (ten for a 75 GB Zim) and `Zims.get_all` did that
on the event loop; and python-libzim never releases the GIL, so the estimate
offloaded with asyncio.to_thread still froze the worker.
Everything libzim does for an estimate now runs in a standalone script started
as a subprocess (Sanic workers are daemonic, so a multiprocessing pool cannot
be forked from them). It reads the Zim paths and search string on stdin and
writes the counts on stdout. A Zim it cannot read is reported and left out, as
`Zims.get_all` would leave it out.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Zim searches run in the libzim worker process too
The /api/zim/search endpoints had the same stall as the estimate: they opened
every Zim through Zims.get_all and searched them with libzim, which holds the
GIL, so a thread offload still froze the Sanic worker.
The estimate script is now a general libzim worker with estimate, search and
entries jobs. A search of all Zims sends every Zim's job to one worker
process; the API worker only headlines the returned HTML and looks up tags.
The worker also returns each Zim's metadata so Zim.__json__ does not open the
Zim again on the event loop. Zim.entries_with_tags is split so the tagged
paths come from the database without opening the Zim.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Tagged Zim estimates count from the database, and a cancelled request kills the libzim worker
Review of the earlier commits found two gaps. Estimates with tags selected
still went through Zims.entries_with_tags, which opened every Zim in the API
process, and after the thread offload was removed it did so on the event loop;
they now count the tagged paths in SQL and ask the worker only for metadata.
And Sanic cancels a handler at its response timeout or on disconnect, which
left the worker process running until its stdout pipe filled; the runner now
kills and reaps it on cancellation.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-01 13:02:26 -07:00
|
|
|
from modules.zim import lib, libzim_worker
|
2024-07-30 17:15:01 -06:00
|
|
|
from modules.zim.errors import UnknownZim, UnknownZimEntry
|
2023-05-29 18:04:03 -06:00
|
|
|
from modules.zim.models import TagZimEntry, ZimSubscription
|
|
|
|
|
from wrolpi import tags, flags
|
2024-03-09 16:27:34 -07:00
|
|
|
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
|
2025-12-06 14:49:46 -07:00
|
|
|
from wrolpi.downloader import Download, import_downloads_config, get_download_manager_config
|
2023-05-29 18:04:03 -06:00
|
|
|
from wrolpi.files.models import FileGroup
|
2026-10-08 17:28:13 -06:00
|
|
|
from wrolpi.test.common import skip_ci
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
2026-01-09 14:01:25 -07:00
|
|
|
async def test_get_zim(async_client, test_session, zim_path_factory, refresh_files):
|
2023-05-29 18:04:03 -06:00
|
|
|
zim_path_factory()
|
2026-01-09 14:01:25 -07:00
|
|
|
await refresh_files()
|
2023-05-29 18:04:03 -06:00
|
|
|
|
2025-12-10 21:16:10 -07:00
|
|
|
assert lib.get_zim(test_session, 1)
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
with pytest.raises(UnknownZim):
|
2025-12-10 21:16:10 -07:00
|
|
|
assert lib.get_zim(test_session, 2)
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
2026-01-09 14:01:25 -07:00
|
|
|
async def test_zim_get_entry(async_client, test_session, zim_path_factory, refresh_files):
|
2023-05-29 18:04:03 -06:00
|
|
|
zim_path_factory()
|
2026-01-09 14:01:25 -07:00
|
|
|
await refresh_files()
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
entry: Entry = lib.get_entry('one', 1)
|
|
|
|
|
assert entry.path == 'one'
|
|
|
|
|
assert entry.title == 'One'
|
|
|
|
|
assert bytes(entry.get_item().content).decode('UTF-8') == '''
|
|
|
|
|
<html>
|
|
|
|
|
<head>
|
|
|
|
|
<title>One</title>
|
|
|
|
|
</head>
|
|
|
|
|
<body>
|
|
|
|
|
<h1>This is the first item</h1>
|
|
|
|
|
</body>
|
|
|
|
|
</html>
|
|
|
|
|
'''
|
|
|
|
|
|
|
|
|
|
entry: Entry = lib.get_entry('home', 1)
|
|
|
|
|
assert entry.path == 'home'
|
|
|
|
|
assert entry.title == 'Homepage'
|
|
|
|
|
|
|
|
|
|
|
2024-07-30 17:15:01 -06:00
|
|
|
@pytest.mark.asyncio
|
2024-09-15 15:33:07 -06:00
|
|
|
async def test_zim_bad_entry(async_client, test_session, test_zim, tag_factory):
|
2024-08-15 19:32:01 -06:00
|
|
|
tag1 = await tag_factory()
|
2024-07-30 17:15:01 -06:00
|
|
|
with pytest.raises(UnknownZimEntry):
|
2025-12-10 21:16:10 -07:00
|
|
|
test_zim.tag_entry(test_session, tag1.name, 'path does not exist')
|
2024-07-30 17:15:01 -06:00
|
|
|
|
|
|
|
|
|
2023-05-29 18:04:03 -06:00
|
|
|
@pytest.mark.asyncio
|
2024-09-15 15:33:07 -06:00
|
|
|
async def test_zim_get_entries_tags(async_client, test_session, test_zim, tag_factory):
|
2024-08-15 19:32:01 -06:00
|
|
|
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
|
2025-12-10 21:16:10 -07:00
|
|
|
test_zim.tag_entry(test_session, tag1.name, 'one')
|
|
|
|
|
test_zim.tag_entry(test_session, tag2.name, 'one')
|
|
|
|
|
test_zim.tag_entry(test_session, tag1.name, 'two')
|
2023-05-29 18:04:03 -06:00
|
|
|
test_session.commit()
|
|
|
|
|
|
db: request-path library functions use the request's connection (#774)
* db: request-path library functions use the request's connection
Library functions an API request reaches now take the request's Session (first argument) instead of
opening their own session or cursor, so a request needs one SQLite connection. Raw SQL runs through the
new `wrolpi.db.session_curs(session)`, a cursor on the Session's connection; `RequestDB.curs()` uses it.
- Search: files, videos, archives, flasher, and docs search share the request connection for the total,
the page of ids, and the page's models (3 connections -> 1).
- Zim search, including tag-filtered search and estimates (2 -> 1).
- Channel detail (`Channel.get_statistics` reads on the Channel's own Session) and the channel list.
- Video detail, edit, description, captions, comments and transcode. `get_video_for_app` and
`update_video` no longer commit; their handlers use `request.ctx.db.write()`, so editing a Video and
reading it back is one transaction.
- Tags (list, overlaps, recent), file statistics and estimates, other estimates, bulk tag preview, the
download list, retry, kill and the status summary, and docs/videos/archive statistics.
Not converted: file operations that refresh files inside the request (delete, rename, upload, channel
and collection tag/move) commit page by page in code shared with the file worker; the cached zim entry
lookup; config import.
The connection-count tests now assert one connection for every endpoint they cover (no xfails left), and
cover the videos and archive statistics endpoints, which had no API test.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
* downloads: startup retries in a write session of its own
start_single_tasks still called retry_downloads() without a Session after it began requiring one, so any
start after a completed refresh raised TypeError there: incomplete downloads were not reset and the
download_on_startup enable after it never ran. Startup is not a request, so
DownloadManager.retry_downloads_on_startup() opens and commits its own short write transaction.
Tested through the method, not start_single_tasks: importing main registers its startup listeners on the
shared app, and every later test in that worker then runs them.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 11:05:20 -06:00
|
|
|
assert lib.get_entries_tags(test_session, ['one', 'two', 'home'], 1) \
|
2023-05-29 18:04:03 -06:00
|
|
|
== dict(one=['tag1', 'tag2'], two=['tag1'], home=[])
|
|
|
|
|
|
|
|
|
|
|
2024-07-30 17:15:01 -06:00
|
|
|
@pytest.mark.asyncio
|
2024-09-15 15:33:07 -06:00
|
|
|
async def test_tag_zim_entry_model(async_client, test_session, zim_factory, tag_factory):
|
2024-08-15 19:32:01 -06:00
|
|
|
tag = await tag_factory()
|
2025-12-10 21:16:10 -07:00
|
|
|
tze = zim_factory('zim1').tag_entry(test_session, tag.name, 'one')
|
2024-07-30 17:15:01 -06:00
|
|
|
# Test TagZimEntry.__repr__
|
|
|
|
|
assert repr(tze) == "<TagZimEntry tag='one' zim=zim1 zim_entry=one>"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
2024-09-15 15:33:07 -06:00
|
|
|
async def test_zim_entries_with_tags(async_client, test_session, zim_factory, tag_factory):
|
2026-10-27 16:00:15 -06:00
|
|
|
"""Can fetch the Entries of a specific Zim that have Tags."""
|
2024-08-15 19:32:01 -06:00
|
|
|
tag1, tag2 = await tag_factory(), await tag_factory()
|
2024-07-30 17:15:01 -06:00
|
|
|
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
|
2026-10-27 16:00:15 -06:00
|
|
|
# Tag Zim1 with both tags. Zim2 shares one of the tags, but its Entry is not returned for Zim1.
|
2025-12-10 21:16:10 -07:00
|
|
|
zim1.tag_entry(test_session, tag1.name, 'one')
|
|
|
|
|
zim1.tag_entry(test_session, tag2.name, 'two')
|
|
|
|
|
zim2.tag_entry(test_session, tag2.name, 'home')
|
2024-07-30 17:15:01 -06:00
|
|
|
test_session.commit()
|
|
|
|
|
|
|
|
|
|
# Get entries from zim1 only.
|
2025-12-10 21:16:10 -07:00
|
|
|
entries = zim1.entries_with_tags(test_session, [tag1.name, ])
|
2024-07-30 17:15:01 -06:00
|
|
|
assert len(entries) == 1
|
|
|
|
|
assert entries[0].path == 'one'
|
2025-12-10 21:16:10 -07:00
|
|
|
entries = zim1.entries_with_tags(test_session, [tag2.name, ])
|
2024-07-30 17:15:01 -06:00
|
|
|
assert len(entries) == 1
|
|
|
|
|
assert entries[0].path == 'two'
|
|
|
|
|
|
|
|
|
|
|
Zim estimates and searches run in a libzim worker process so a cold Zim cannot stall the API worker (#928)
* Zim search estimates run in a separate process so a cold estimate cannot stall the API worker
The global search estimates every auto-search Zim on each keystroke. On a
large library that estimate took tens of seconds when the Zims' pages were not
cached, and every other request routed to the same Sanic worker, such as the
channel suggestions, waited behind it. Two things held the worker: opening a
large Zim cold takes seconds (ten for a 75 GB Zim) and `Zims.get_all` did that
on the event loop; and python-libzim never releases the GIL, so the estimate
offloaded with asyncio.to_thread still froze the worker.
Everything libzim does for an estimate now runs in a standalone script started
as a subprocess (Sanic workers are daemonic, so a multiprocessing pool cannot
be forked from them). It reads the Zim paths and search string on stdin and
writes the counts on stdout. A Zim it cannot read is reported and left out, as
`Zims.get_all` would leave it out.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Zim searches run in the libzim worker process too
The /api/zim/search endpoints had the same stall as the estimate: they opened
every Zim through Zims.get_all and searched them with libzim, which holds the
GIL, so a thread offload still froze the Sanic worker.
The estimate script is now a general libzim worker with estimate, search and
entries jobs. A search of all Zims sends every Zim's job to one worker
process; the API worker only headlines the returned HTML and looks up tags.
The worker also returns each Zim's metadata so Zim.__json__ does not open the
Zim again on the event loop. Zim.entries_with_tags is split so the tagged
paths come from the database without opening the Zim.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Tagged Zim estimates count from the database, and a cancelled request kills the libzim worker
Review of the earlier commits found two gaps. Estimates with tags selected
still went through Zims.entries_with_tags, which opened every Zim in the API
process, and after the thread offload was removed it did so on the event loop;
they now count the tagged paths in SQL and ask the worker only for metadata.
And Sanic cancels a handler at its response timeout or on disconnect, which
left the worker process running until its stdout pipe filled; the runner now
kills and reaps it on cancellation.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-01 13:02:26 -07:00
|
|
|
def test_unique_paths():
|
2023-05-29 18:04:03 -06:00
|
|
|
paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear',
|
|
|
|
|
'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear',
|
|
|
|
|
'A/Chenille_prickly-pear', 'A/Coastal_Prickly_Pear', 'A/Coastal_prickly_pear',
|
|
|
|
|
'A/Prickly_pears_in_South_Africa', 'A/Prickly_Pear_Cays', 'A/Prickly_pears_in_Australia', 'A/Opuntia',
|
|
|
|
|
'A/Prickly_Pear_(British_Virgin_Islands)', 'A/Prickly_Pear_Island', 'A/Prickly_Pears_(film)',
|
|
|
|
|
'A/Invasive_succulent_plants_in_South_Africa', 'A/Cactoblastis_Memorial_Hall', 'A/Thomas_Harvey_Johnston']
|
Zim estimates and searches run in a libzim worker process so a cold Zim cannot stall the API worker (#928)
* Zim search estimates run in a separate process so a cold estimate cannot stall the API worker
The global search estimates every auto-search Zim on each keystroke. On a
large library that estimate took tens of seconds when the Zims' pages were not
cached, and every other request routed to the same Sanic worker, such as the
channel suggestions, waited behind it. Two things held the worker: opening a
large Zim cold takes seconds (ten for a 75 GB Zim) and `Zims.get_all` did that
on the event loop; and python-libzim never releases the GIL, so the estimate
offloaded with asyncio.to_thread still froze the worker.
Everything libzim does for an estimate now runs in a standalone script started
as a subprocess (Sanic workers are daemonic, so a multiprocessing pool cannot
be forked from them). It reads the Zim paths and search string on stdin and
writes the counts on stdout. A Zim it cannot read is reported and left out, as
`Zims.get_all` would leave it out.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Zim searches run in the libzim worker process too
The /api/zim/search endpoints had the same stall as the estimate: they opened
every Zim through Zims.get_all and searched them with libzim, which holds the
GIL, so a thread offload still froze the Sanic worker.
The estimate script is now a general libzim worker with estimate, search and
entries jobs. A search of all Zims sends every Zim's job to one worker
process; the API worker only headlines the returned HTML and looks up tags.
The worker also returns each Zim's metadata so Zim.__json__ does not open the
Zim again on the event loop. Zim.entries_with_tags is split so the tagged
paths come from the database without opening the Zim.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Tagged Zim estimates count from the database, and a cancelled request kills the libzim worker
Review of the earlier commits found two gaps. Estimates with tags selected
still went through Zims.entries_with_tags, which opened every Zim in the API
process, and after the thread offload was removed it did so on the event loop;
they now count the tagged paths in SQL and ask the worker only for metadata.
And Sanic cancels a handler at its response timeout or on disconnect, which
left the worker process running until its stdout pipe filled; the runner now
kills and reaps it on cancellation.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-01 13:02:26 -07:00
|
|
|
assert libzim_worker.unique_paths(*paths) == ['A/Prickly_Pear',
|
|
|
|
|
'A/Arborescent_Prickly_Pear',
|
|
|
|
|
'A/Beavertail_prickly_pear',
|
|
|
|
|
'A/Brittle_prickly-pear',
|
|
|
|
|
'A/Chenille_prickly-pear',
|
|
|
|
|
'A/Coastal_Prickly_Pear',
|
|
|
|
|
'A/Prickly_pears_in_South_Africa',
|
|
|
|
|
'A/Prickly_Pear_Cays',
|
|
|
|
|
'A/Prickly_pears_in_Australia',
|
|
|
|
|
'A/Opuntia',
|
|
|
|
|
'A/Prickly_Pear_(British_Virgin_Islands)',
|
|
|
|
|
'A/Prickly_Pear_Island',
|
|
|
|
|
'A/Prickly_Pears_(film)',
|
|
|
|
|
'A/Invasive_succulent_plants_in_South_Africa',
|
|
|
|
|
'A/Cactoblastis_Memorial_Hall',
|
|
|
|
|
'A/Thomas_Harvey_Johnston']
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
|
2024-08-15 19:32:01 -06:00
|
|
|
@pytest.mark.asyncio
|
2024-09-15 15:33:07 -06:00
|
|
|
async def test_zim_tags_config(async_client, test_session, test_directory, test_zim, tag_factory, test_tags_config,
|
2024-12-11 11:52:33 -07:00
|
|
|
fake_now, await_switches):
|
2023-05-29 18:04:03 -06:00
|
|
|
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
|
|
|
|
|
config = tags.get_tags_config()
|
|
|
|
|
assert not config.tag_zims
|
|
|
|
|
|
|
|
|
|
# Tag three Zim entries.
|
2024-08-15 19:32:01 -06:00
|
|
|
tag1, tag2 = await tag_factory('Tag1'), await tag_factory('Tag2')
|
2025-12-10 21:16:10 -07:00
|
|
|
test_zim.tag_entry(test_session, tag1.name, 'one')
|
|
|
|
|
test_zim.tag_entry(test_session, tag2.name, 'one')
|
|
|
|
|
test_zim.tag_entry(test_session, tag1.name, 'two')
|
2023-05-29 18:04:03 -06:00
|
|
|
test_session.commit()
|
2024-09-15 15:33:07 -06:00
|
|
|
await await_switches()
|
2023-05-29 18:04:03 -06:00
|
|
|
# Tags are created in empty database.
|
|
|
|
|
assert {i[0] for i in test_session.query(tags.Tag.id)} == {1, 2}
|
|
|
|
|
|
|
|
|
|
config = tags.get_tags_config()
|
|
|
|
|
assert config.tag_zims == [
|
|
|
|
|
['Tag1', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
|
|
|
|
|
['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
|
|
|
|
|
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
# Remove a tag, the config should change.
|
2025-12-10 21:16:10 -07:00
|
|
|
test_zim.untag_entry(test_session, tag1.name, 'one')
|
2023-05-29 18:04:03 -06:00
|
|
|
test_session.commit()
|
2024-09-15 15:33:07 -06:00
|
|
|
await await_switches()
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
config = tags.get_tags_config()
|
|
|
|
|
assert config.tag_zims == [
|
|
|
|
|
['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
|
|
|
|
|
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
|
|
|
|
|
]
|
|
|
|
|
|
2025-11-29 09:14:32 -07:00
|
|
|
# Delete all TagZimEntry(s) first (due to FK constraint), then Tags so they are recreated.
|
2023-05-29 18:04:03 -06:00
|
|
|
test_session.query(TagZimEntry).delete()
|
2025-11-29 09:14:32 -07:00
|
|
|
test_session.query(tags.Tag).delete()
|
2023-05-29 18:04:03 -06:00
|
|
|
test_session.commit()
|
|
|
|
|
|
|
|
|
|
tags.import_tags_config()
|
|
|
|
|
|
|
|
|
|
tag_zim_entries = test_session.query(TagZimEntry) \
|
|
|
|
|
.order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry)
|
|
|
|
|
assert [(i.tag.name, i.zim.path, i.zim_entry) for i in tag_zim_entries] \
|
|
|
|
|
== [('Tag2', test_zim.path, 'one'), ('Tag1', test_zim.path, 'two')]
|
|
|
|
|
|
Postgres -> SQLite hard cutover: the database moves into the media directory.
The database now lives at <media>/config/wrolpi.db so a library (files,
configs, AND database) travels with the drive between WROLPi systems.
Core:
- wrolpi/db.py: lazy SQLite engine factory (WAL, busy_timeout=30s,
foreign_keys=ON); get_db_curs yields a plain sqlite3.Cursor; native
sqlite3 paramstyle everywhere (no compatibility shims); global sqlite3
adapters keep raw-SQL datetime formats identical to the ORM's.
- wrolpi/fts.py: FTS5 external-content search replacing tsvector/GIN.
Injection-proof websearch->FTS5 translator, bm25(10,4,2,1) reproducing
A/B/C/D weighting, fast(abc)/deep column filtering, snippet headlines.
Stemming contract pinned by dedicated tests (watch/watches/watched/
watching are interchangeable).
- wrolpi/schema_ddl.py: SQLite triggers shared verbatim by the alembic
baseline and test DBs; fixes the channel video_count DELETE/UPDATE
bugs and a latent total_size bug (it used the video id as channel id).
- wrolpi/db_bootstrap.py: startup auto-create/migrate; refuses databases
written by a newer WROLPi (db_version_mismatch); SQLite 3.40 floor;
network-filesystem refusal; never creates/migrates/trusts a database
beneath an unmounted production media directory (shadow-DB guard).
- Alembic: 85 Postgres-era migrations squashed into one baseline;
revision ids are dates (2026_07_10_1350) so filenames self-sort and
alembic_version says when a schema was created; render_as_batch=True.
- All raw Postgres SQL rewritten (ANY->json_each IN, array_agg @> ->
HAVING COUNT(DISTINCT), unnest->executemany/UPDATE..FROM,
generate_series histograms->strftime GROUP BY, DISTINCT ON->
row_number(), writable CTEs->two statements, jsonb_*->json_*, explicit
NULLS FIRST/LAST, LIMIT-before-OFFSET). Simple statements converted
to SQLAlchemy instead (flag upsert, download retry/summary/tag checks,
channel-by-tag estimate, Directory delete).
- Config imports count skipped entries; background dumps are blocked
after a lossy import so partial imports can never overwrite YAML
configs (they are the sole source of truth for rebuilds now).
- Search tolerates explicit JSON nulls for limit/offset (Postgres
treated OFFSET NULL as 0; SQLite errors).
- file_group.id/doc_section.id render as INTEGER PRIMARY KEY on SQLite
(BIGINT is not a rowid alias; FTS5 content_rowid requires it).
Ops teardown: docker-compose db service, CI postgres sidecars,
initialize_api_db.sh, controller postgresql service, psycopg2-binary all
removed. upgrade.sh stops/disables postgresql (data is NOT deleted;
`systemctl enable --now postgresql` restores it); systemd units no
longer Want postgresql.service.
Existing-user upgrade: first boot creates+migrates the DB, configs
import (tags recreate their FileGroups from disk), a refresh rebuilds
the library index. QA'd on the 10.0.0.8 RPi: rebuild matched the
Postgres baseline exactly; survived an uncontrolled hard reboot with
clean integrity checks.
Accepted behavior changes: FTS5 has no stopwords; bm25 length
normalization shifts some rank orders; fast-path d_headline is
un-highlighted; all-negative queries match nothing.
Deferred: Debian/trixie package lists (incl. sqlite3 CLI), Cypress
re-baseline, VACUUM INTO snapshots, exFAT/NTFS warning, auto-kicked
rebuild refresh with progress UX.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-12 13:59:20 -06:00
|
|
|
# Tags were recreated. (SQLite reuses row ids after DELETE, so assert by name rather than by id.)
|
|
|
|
|
assert {i.name for i in test_session.query(tags.Tag)} == {'Tag1', 'Tag2'}
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
|
2026-10-08 17:28:13 -06:00
|
|
|
@skip_ci
|
2023-05-29 18:04:03 -06:00
|
|
|
@pytest.mark.asyncio
|
2024-12-11 11:52:33 -07:00
|
|
|
async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock,
|
|
|
|
|
await_switches):
|
2025-12-10 21:16:10 -07:00
|
|
|
await lib.subscribe(test_session, 'Wikipedia (with images)', 'es')
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
# Downloading the catalog should lead to a new Zim file being downloaded.
|
|
|
|
|
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim')
|
|
|
|
|
recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
|
|
|
|
|
recurring_download: Download
|
|
|
|
|
once_download: Download
|
|
|
|
|
assert recurring_download.attempts == 1 and recurring_download.status == 'complete', \
|
|
|
|
|
'Recurring download should have succeeded'
|
|
|
|
|
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
|
|
|
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
|
|
|
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
|
|
|
|
|
assert once_download.attempts == 1
|
2025-03-21 22:12:49 -06:00
|
|
|
# All files downloaded were indexed and are Zim files.
|
|
|
|
|
assert all(i.model == 'zim' and i.indexed is True for i in test_session.query(FileGroup))
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
# Download catalog again, should not re-download the existing file.
|
|
|
|
|
recurring_download.status = 'new'
|
|
|
|
|
test_session.commit()
|
|
|
|
|
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim',
|
|
|
|
|
download_file_side_effect=Exception('should not be called twice'))
|
|
|
|
|
recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
|
|
|
|
|
assert recurring_download.attempts == 2 and recurring_download.status == 'complete', \
|
|
|
|
|
'Recurring download should have happened again'
|
|
|
|
|
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
|
|
|
|
|
# New file was modeled into a FileGroup.
|
|
|
|
|
assert test_session.query(FileGroup).count() == 1
|
|
|
|
|
# No outdated files exist.
|
|
|
|
|
assert not flags.outdated_zims.is_set()
|
|
|
|
|
|
|
|
|
|
# Download catalog again, should fetch the latest file.
|
|
|
|
|
recurring_download.status = 'new'
|
|
|
|
|
test_session.commit()
|
|
|
|
|
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2024-01.zim',
|
|
|
|
|
download_info=DownloadFileInfo(
|
|
|
|
|
status=HTTPStatus.OK,
|
|
|
|
|
name='wikipedia_es_all_maxi_2024-01.zim',
|
|
|
|
|
size=len(test_zim_bytes),
|
|
|
|
|
),
|
|
|
|
|
hrefs=['?C=N;O=D', '?C=M;O=A', '?C=S;O=A', '?C=D;O=A', '/zim/',
|
|
|
|
|
'wikipedia_es_all_maxi_2023-05.zim',
|
|
|
|
|
'wikipedia_es_all_maxi_2023-06.zim',
|
|
|
|
|
'wikipedia_es_all_maxi_2024-01.zim', # The new Zim.
|
|
|
|
|
'wikipedia_es_all_nopic_2023-05.zim',
|
|
|
|
|
'wikipedia_es_all_nopic_2023-06.zim', ]
|
|
|
|
|
)
|
|
|
|
|
recurring_download, first_download, second_download = test_session.query(Download).order_by(Download.id).all()
|
|
|
|
|
assert recurring_download.attempts == 3 and recurring_download.status == 'complete', \
|
|
|
|
|
'Recurring download should have happened again'
|
|
|
|
|
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
|
|
|
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
|
|
|
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
|
|
|
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').is_file()
|
|
|
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').stat().st_size
|
|
|
|
|
# New file was modeled.
|
|
|
|
|
assert test_session.query(FileGroup).count() == 2
|
|
|
|
|
# But old file is now outdated.
|
|
|
|
|
assert flags.outdated_zims.is_set()
|
|
|
|
|
|
2025-03-21 22:12:49 -06:00
|
|
|
# All files downloaded were indexed and are Zim files.
|
|
|
|
|
assert all(i.model == 'zim' and i.indexed is True for i in test_session.query(FileGroup))
|
|
|
|
|
|
2023-05-29 18:04:03 -06:00
|
|
|
# Delete the outdated file.
|
|
|
|
|
await lib.remove_outdated_zim_files()
|
|
|
|
|
assert test_session.query(FileGroup).count() == 1
|
|
|
|
|
# No longer outdated
|
|
|
|
|
assert not flags.outdated_zims.is_set()
|
|
|
|
|
|
|
|
|
|
|
2024-08-15 19:32:01 -06:00
|
|
|
@pytest.mark.asyncio
|
2024-12-11 11:52:33 -07:00
|
|
|
async def test_zim_tag_migration(await_switches, test_session, test_directory, zim_factory, tag_factory,
|
2024-09-17 18:04:00 -07:00
|
|
|
test_tags_config):
|
2023-05-29 18:04:03 -06:00
|
|
|
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
|
|
|
|
|
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
|
|
|
|
|
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
|
|
|
|
|
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
|
2024-08-15 19:32:01 -06:00
|
|
|
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
|
2023-05-29 18:04:03 -06:00
|
|
|
# Tag outdated Zim entries.
|
2025-12-10 21:16:10 -07:00
|
|
|
zim1.tag_entry(test_session, tag1.name, 'home')
|
|
|
|
|
zim1.tag_entry(test_session, tag2.name, 'one')
|
2023-05-29 18:04:03 -06:00
|
|
|
# This should not be migrated.
|
2025-12-10 21:16:10 -07:00
|
|
|
zim2.tag_entry(test_session, tag2.name, 'two')
|
2023-05-29 18:04:03 -06:00
|
|
|
# Tag the latest Zim as well.
|
2025-12-10 21:16:10 -07:00
|
|
|
zim3.tag_entry(test_session, tag2.name, 'one')
|
2023-05-29 18:04:03 -06:00
|
|
|
test_session.commit()
|
|
|
|
|
assert test_session.query(TagZimEntry).count() == 4
|
2024-09-15 15:33:07 -06:00
|
|
|
await await_switches()
|
2024-09-26 09:13:29 -06:00
|
|
|
assert tags.get_tags_config().successful_import is True
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
# All Zims have been tagged.
|
|
|
|
|
tags_config = tags.get_tags_config()
|
2024-09-26 09:13:29 -06:00
|
|
|
tags_config.dump_config()
|
2023-05-29 18:04:03 -06:00
|
|
|
assert 'wikipedia_en_all_maxi_2020-01.zim' in (config_text := tags_config.get_file().read_text())
|
|
|
|
|
assert 'wikipedia_en_all_maxi_2020-02.zim' in config_text
|
|
|
|
|
assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
|
|
|
|
|
|
|
|
|
|
# Zim was deleted, as well as it's ZimTagEntry(s).
|
|
|
|
|
zim1.delete()
|
2024-09-26 09:13:29 -06:00
|
|
|
tags.import_tags_config()
|
2023-05-29 18:04:03 -06:00
|
|
|
|
|
|
|
|
tag_zim_entries = sorted([(i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry)])
|
|
|
|
|
assert tag_zim_entries == [
|
|
|
|
|
('tag1', zim3.path, 'home'),
|
|
|
|
|
('tag2', zim2.path, 'two'),
|
|
|
|
|
('tag2', zim3.path, 'one'), # Both "one" tags were combined.
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
2026-10-04 18:04:11 -06:00
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_remove_outdated_zim_files_migrates_tags(await_switches, test_session, test_directory, zim_factory,
|
|
|
|
|
tag_factory, test_tags_config):
|
|
|
|
|
"""Deleting outdated Zims moves their entry tags to the newest Zim of the same name.
|
|
|
|
|
|
|
|
|
|
Deleting the Zim cascade-deletes its TagZimEntry rows, and the next tags save writes them out of the config,
|
|
|
|
|
so the tags must be moved before the files are deleted."""
|
|
|
|
|
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # Outdated.
|
|
|
|
|
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim') # Outdated.
|
|
|
|
|
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
|
|
|
|
|
other = zim_factory('wikibooks_en_all_maxi_2020-01.zim') # A different Zim, not outdated.
|
|
|
|
|
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
|
|
|
|
|
zim1.tag_entry(test_session, tag1.name, 'home')
|
|
|
|
|
zim1.tag_entry(test_session, tag2.name, 'one')
|
|
|
|
|
zim2.tag_entry(test_session, tag2.name, 'two')
|
|
|
|
|
zim3.tag_entry(test_session, tag2.name, 'one') # Already on the latest Zim; must not be duplicated.
|
|
|
|
|
other.tag_entry(test_session, tag1.name, 'home')
|
|
|
|
|
test_session.commit()
|
|
|
|
|
zim3_path, other_path = zim3.path, other.path
|
|
|
|
|
await await_switches()
|
|
|
|
|
|
|
|
|
|
assert await lib.remove_outdated_zim_files(test_directory) == 2
|
|
|
|
|
await await_switches()
|
|
|
|
|
|
|
|
|
|
test_session.expire_all()
|
|
|
|
|
tag_zim_entries = sorted((i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry))
|
|
|
|
|
assert tag_zim_entries == [
|
|
|
|
|
('tag1', other_path, 'home'),
|
|
|
|
|
('tag1', zim3_path, 'home'),
|
|
|
|
|
('tag2', zim3_path, 'one'),
|
|
|
|
|
('tag2', zim3_path, 'two'),
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
# The tags config was saved with the migrated tags, so a later import keeps them.
|
|
|
|
|
config_text = tags.get_tags_config().get_file().read_text()
|
|
|
|
|
assert 'wikipedia_en_all_maxi_2020-01.zim' not in config_text
|
|
|
|
|
assert 'wikipedia_en_all_maxi_2020-02.zim' not in config_text
|
|
|
|
|
assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
|
|
|
|
|
|
2023-05-29 18:04:03 -06:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
|
'url,expected',
|
|
|
|
|
[
|
|
|
|
|
('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_maxi_', ('Wikipedia (with images)', 'arz')),
|
|
|
|
|
('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_nopic_', ('Wikipedia (no images)', 'arz')),
|
|
|
|
|
('https://download.kiwix.org/zim/wikipedia/wikipedia_ar_all_mini_', ('Wikipedia (mini)', 'ar')),
|
|
|
|
|
('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_maxi_', ('Wikiversity (with images)', 'zh')),
|
|
|
|
|
('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_nopic_', ('Wikiversity (no images)', 'zh')),
|
|
|
|
|
('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_maxi_', ('Wikitionary (with images)', 'eo')),
|
|
|
|
|
('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_nopic_', ('Wikitionary (no images)', 'eo')),
|
|
|
|
|
('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_nopic_', ('Wikibooks (no images)', 'es')),
|
|
|
|
|
('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_maxi_', ('Wikibooks (with images)', 'es')),
|
|
|
|
|
('https://download.kiwix.org/zim/wikisource/wikisource_bn_all_maxi_', ('Wikisource', 'bn')),
|
|
|
|
|
('https://download.kiwix.org/zim/vikidia/vikidia_ru_all_maxi_', ('Vikidia', 'ru')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_en_all_',
|
|
|
|
|
('Raspberry Pi (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_en_all_',
|
|
|
|
|
('Stackoverflow (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/ja.stackoverflow.com_ja_all_',
|
|
|
|
|
('Stackoverflow (Stack Exchange)', 'ja')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/superuser.com_en_all_', ('Superuser (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/ifixit/ifixit_nl_all_', ('iFixit', 'nl')),
|
|
|
|
|
('https://download.kiwix.org/zim/gutenberg/gutenberg_ang_all_', ('Gutenberg', 'ang')),
|
2023-10-09 16:46:46 -06:00
|
|
|
('https://download.kiwix.org/zim/stack_exchange/ham.stackexchange.com_en_all_',
|
2024-02-12 12:40:56 -07:00
|
|
|
('Amateur Radio (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/diy.stackexchange.com_en_all_',
|
|
|
|
|
('DIY (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/electronics.stackexchange.com_en_all_',
|
|
|
|
|
('Electronics (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/unix.stackexchange.com_en_all_',
|
|
|
|
|
('Unix (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/askubuntu.com_en_all_',
|
|
|
|
|
('Ask Ubuntu (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/bicycles.stackexchange.com_en_all_',
|
|
|
|
|
('Bicycles (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/biology.stackexchange.com_en_all_',
|
|
|
|
|
('Biology (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/arduino.stackexchange.com_en_all_',
|
|
|
|
|
('Arduino (Stack Exchange)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/3dprinting.stackexchange.com_en_all_',
|
|
|
|
|
('3D Printing (Stack Exchange)', 'en')),
|
2024-02-13 14:07:34 -07:00
|
|
|
('https://download.kiwix.org/zim/other/mdwiki_en_all_',
|
|
|
|
|
('MD Wiki (Wiki Project Med Foundation)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/wikipedia/wikipedia_bpy_medicine_',
|
|
|
|
|
('WikiMed Medical Encyclopedia', 'bpy')),
|
|
|
|
|
('https://download.kiwix.org/zim/other/zimgit-post-disaster_en_',
|
|
|
|
|
('Post Disaster Resource Library', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/zimit/fas-military-medicine_en_',
|
|
|
|
|
('Military Medicine', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/other/wikem_en_all_maxi_',
|
|
|
|
|
('WikEm (Global Emergency Medicine Wiki)', 'en')),
|
|
|
|
|
('https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_en_all_',
|
|
|
|
|
('Health (Stack Exchange)', 'en')),
|
2025-03-03 16:32:35 -07:00
|
|
|
('https://download.kiwix.org/zim/other/khanacademy_en_all_',
|
|
|
|
|
('Khan Academy', 'en')),
|
2023-05-29 18:04:03 -06:00
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
def test_zim_download_url_to_name(url, expected):
|
|
|
|
|
assert lib.zim_download_url_to_name(url) == expected
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
2024-11-28 13:28:23 -07:00
|
|
|
async def test_zim_subscription_download_import(async_client, test_session):
|
2023-05-29 18:04:03 -06:00
|
|
|
# Subscription creates a ZimSubscription and Download
|
2025-12-10 21:16:10 -07:00
|
|
|
await lib.subscribe(test_session, 'Wikisource', 'en')
|
2023-05-29 18:04:03 -06:00
|
|
|
# Add a once-download. This should not be associated with a ZimSubscription.
|
|
|
|
|
test_session.add(
|
|
|
|
|
Download(url='https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim', status='complete'))
|
|
|
|
|
test_session.commit()
|
|
|
|
|
download = test_session.query(Download).filter(Download.frequency.isnot(None)).one()
|
|
|
|
|
assert download.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
|
|
|
|
|
|
2025-12-06 14:47:52 -07:00
|
|
|
# Save config file synchronously (dump_config() + save()), delete all entries.
|
|
|
|
|
config = get_download_manager_config()
|
|
|
|
|
config.dump_config()
|
|
|
|
|
config.save()
|
2023-05-29 18:04:03 -06:00
|
|
|
test_session.query(Download).delete()
|
|
|
|
|
test_session.query(ZimSubscription).delete()
|
|
|
|
|
test_session.commit()
|
|
|
|
|
|
|
|
|
|
for _ in range(1): # Import twice.
|
|
|
|
|
# Importing the config restores the Download and ZimSubscription.
|
2025-12-16 19:59:28 -07:00
|
|
|
await import_downloads_config()
|
Postgres -> SQLite hard cutover: the database moves into the media directory.
The database now lives at <media>/config/wrolpi.db so a library (files,
configs, AND database) travels with the drive between WROLPi systems.
Core:
- wrolpi/db.py: lazy SQLite engine factory (WAL, busy_timeout=30s,
foreign_keys=ON); get_db_curs yields a plain sqlite3.Cursor; native
sqlite3 paramstyle everywhere (no compatibility shims); global sqlite3
adapters keep raw-SQL datetime formats identical to the ORM's.
- wrolpi/fts.py: FTS5 external-content search replacing tsvector/GIN.
Injection-proof websearch->FTS5 translator, bm25(10,4,2,1) reproducing
A/B/C/D weighting, fast(abc)/deep column filtering, snippet headlines.
Stemming contract pinned by dedicated tests (watch/watches/watched/
watching are interchangeable).
- wrolpi/schema_ddl.py: SQLite triggers shared verbatim by the alembic
baseline and test DBs; fixes the channel video_count DELETE/UPDATE
bugs and a latent total_size bug (it used the video id as channel id).
- wrolpi/db_bootstrap.py: startup auto-create/migrate; refuses databases
written by a newer WROLPi (db_version_mismatch); SQLite 3.40 floor;
network-filesystem refusal; never creates/migrates/trusts a database
beneath an unmounted production media directory (shadow-DB guard).
- Alembic: 85 Postgres-era migrations squashed into one baseline;
revision ids are dates (2026_07_10_1350) so filenames self-sort and
alembic_version says when a schema was created; render_as_batch=True.
- All raw Postgres SQL rewritten (ANY->json_each IN, array_agg @> ->
HAVING COUNT(DISTINCT), unnest->executemany/UPDATE..FROM,
generate_series histograms->strftime GROUP BY, DISTINCT ON->
row_number(), writable CTEs->two statements, jsonb_*->json_*, explicit
NULLS FIRST/LAST, LIMIT-before-OFFSET). Simple statements converted
to SQLAlchemy instead (flag upsert, download retry/summary/tag checks,
channel-by-tag estimate, Directory delete).
- Config imports count skipped entries; background dumps are blocked
after a lossy import so partial imports can never overwrite YAML
configs (they are the sole source of truth for rebuilds now).
- Search tolerates explicit JSON nulls for limit/offset (Postgres
treated OFFSET NULL as 0; SQLite errors).
- file_group.id/doc_section.id render as INTEGER PRIMARY KEY on SQLite
(BIGINT is not a rowid alias; FTS5 content_rowid requires it).
Ops teardown: docker-compose db service, CI postgres sidecars,
initialize_api_db.sh, controller postgresql service, psycopg2-binary all
removed. upgrade.sh stops/disables postgresql (data is NOT deleted;
`systemctl enable --now postgresql` restores it); systemd units no
longer Want postgresql.service.
Existing-user upgrade: first boot creates+migrates the DB, configs
import (tags recreate their FileGroups from disk), a refresh rebuilds
the library index. QA'd on the 10.0.0.8 RPi: rebuild matched the
Postgres baseline exactly; survived an uncontrolled hard reboot with
clean integrity checks.
Accepted behavior changes: FTS5 has no stopwords; bm25 length
normalization shifts some rank orders; fast-path d_headline is
un-highlighted; all-negative queries match nothing.
Deferred: Debian/trixie package lists (incl. sqlite3 CLI), Cypress
re-baseline, VACUUM INTO snapshots, exFAT/NTFS warning, auto-kicked
rebuild refresh with progress UX.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-12 13:59:20 -06:00
|
|
|
# NULLS LAST because SQLite sorts NULLs first (Postgres sorted them last).
|
|
|
|
|
recurring, once = test_session.query(Download).order_by(Download.frequency.nullslast())
|
2023-05-29 18:04:03 -06:00
|
|
|
assert once.url == 'https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim'
|
|
|
|
|
assert test_session.query(Download).count() == 2
|
|
|
|
|
assert recurring.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
|
|
|
|
|
assert recurring.frequency
|
|
|
|
|
subscription: ZimSubscription = test_session.query(ZimSubscription).one()
|
|
|
|
|
assert subscription.download_id == recurring.id
|
|
|
|
|
assert subscription.name == 'Wikisource'
|
|
|
|
|
assert subscription.language == 'en'
|
2024-02-27 18:54:14 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
2026-01-09 14:01:25 -07:00
|
|
|
async def test_zim_modeler(async_client, test_session, zim_factory, refresh_files):
|
2024-02-27 18:54:14 -07:00
|
|
|
"""Zim modeler sets FileGroup title and a_text."""
|
2025-03-21 22:12:49 -06:00
|
|
|
zim = zim_factory('the_zim_title.zim')
|
|
|
|
|
assert zim.file_group.indexed is False
|
2024-02-27 18:54:14 -07:00
|
|
|
test_session.commit()
|
|
|
|
|
|
2026-01-09 14:01:25 -07:00
|
|
|
await refresh_files()
|
2024-02-27 18:54:14 -07:00
|
|
|
|
|
|
|
|
fg: FileGroup = test_session.query(FileGroup).one()
|
2025-03-21 22:12:49 -06:00
|
|
|
assert fg.indexed is True
|
|
|
|
|
assert fg.title == 'the_zim_title.zim'
|
|
|
|
|
assert fg.a_text == 'the zim title zim'
|
|
|
|
|
assert fg.model == 'zim'
|
2024-03-09 16:27:34 -07:00
|
|
|
|
|
|
|
|
|
2025-12-10 21:16:10 -07:00
|
|
|
def test_get_custom_zims_directory(async_client, test_directory, test_wrolpi_config):
|
2024-03-09 16:27:34 -07:00
|
|
|
"""Custom directory can be used for zims directory."""
|
|
|
|
|
assert lib.get_zim_directory() == (test_directory / 'zims')
|
|
|
|
|
|
2024-08-06 18:13:21 -06:00
|
|
|
get_wrolpi_config().zims_destination = 'custom/zims'
|
2024-03-09 16:27:34 -07:00
|
|
|
|
|
|
|
|
assert lib.get_zim_directory() == (test_directory / 'custom/zims')
|