Linking to specific page or section of an EPUB/PDF when user searches.

This commit is contained in:
Roland 2026-04-13 15:15:29 -06:00 • committed by lrnselfreliance
parent 673d851501
commit 8a75cf9057
13 changed files with 598 additions and 16 deletions

View file

@ -0,0 +1,51 @@
"""Create doc_section table for per-chapter/per-page full-text search.
Enables deep-linking from search results into a specific EPUB chapter or PDF page,
and fixes a latent completeness bug in full-text search where FileGroup.d_text is
truncated to ~100KB (so only the first ~30 pages of a big PDF were searchable).
Existing EPUB and PDF FileGroups are marked `indexed = FALSE` so that the regular
refresh pipeline re-runs their extractors and populates doc_section rows. No one-off
script is required. Note: the next refresh on a populated library will be heavier
than usual while every EPUB and PDF is re-extracted.
Revision ID: d5e6f7a8b9c0
Revises: c4d5e6f7a8b9
Create Date: 2026-04-13
"""
from alembic import op
revision = 'd5e6f7a8b9c0'
down_revision = 'c4d5e6f7a8b9'
branch_labels = None
depends_on = None
def upgrade():
op.execute('''
CREATE TABLE doc_section (
id BIGSERIAL PRIMARY KEY,
doc_id INTEGER NOT NULL REFERENCES doc(id) ON DELETE CASCADE,
kind TEXT NOT NULL,
ordinal INTEGER NOT NULL,
label TEXT,
content TEXT,
tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english'::regconfig, COALESCE(content, ''))) STORED
)
''')
op.execute('CREATE INDEX doc_section_doc_id_idx ON doc_section (doc_id, ordinal)')
op.execute('CREATE INDEX doc_section_tsv_idx ON doc_section USING GIN (tsv)')
# Force a re-index of all EPUB and PDF file groups so the refresh pipeline
# populates the new doc_section table.
op.execute('''
UPDATE file_group
SET indexed = FALSE
WHERE mimetype LIKE 'application/epub%'
OR mimetype = 'application/pdf'
''')
def downgrade():
op.execute('DROP TABLE IF EXISTS doc_section')

View file

@ -175,6 +175,9 @@ either expressed or implied, of the FreeBSD Project. -->
let params = URLSearchParams && new URLSearchParams(document.location.search.substring(1));
let url = params && params.get("url") && decodeURIComponent(params.get("url"));
let currentSectionIndex = (params && params.get("loc")) ? params.get("loc") : undefined;
// Optional search term passed in from a search-result deep link. If present we'll
// search the spine item at `loc` and jump to + highlight the first match.
let searchQuery = (params && params.get("q")) ? params.get("q") : null;
let book = ePub(url);
let rendition = book.renderTo("viewer", {
@ -185,6 +188,44 @@ either expressed or implied, of the FreeBSD Project. -->
rendition.display(currentSectionIndex);
// If a search query was provided, try to locate it within the target spine item
// and scroll to / highlight the first match. Any failure is non-fatal — the `loc`
// jump above still lands the user at the right chapter.
if (searchQuery) {
book.ready.then(function () {
try {
let spine = book.spine;
let item;
if (currentSectionIndex !== undefined) {
let n = parseInt(currentSectionIndex, 10);
if (!isNaN(n)) {
item = spine.get(n);
}
}
if (!item && spine.spineItems && spine.spineItems.length) {
item = spine.spineItems[0];
}
if (!item) return;
Promise.resolve(item.load(book.load.bind(book))).then(function () {
let results = item.find(searchQuery) || [];
if (results.length > 0) {
let hit = results[0];
rendition.display(hit.cfi);
try {
rendition.annotations.highlight(hit.cfi);
} catch (e) {
console.warn("highlight failed", e);
}
}
}).finally(function () {
try { item.unload(); } catch (e) { /* ignore */ }
});
} catch (e) {
console.warn("epub deep-link search failed", e);
}
});
}
book.ready.then(() => {
let next = document.getElementById("next");

View file

@ -1,5 +1,5 @@
import React, {useContext, useEffect, useState} from 'react';
import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router";
import {Link, Route, Routes, useLocation, useNavigate, useParams, useSearchParams} from "react-router";
import {Grid, Icon as SIcon, Loader, StatisticLabel, StatisticValue} from "semantic-ui-react";
import {deleteDocs, getDocStatistics, tagFileGroup, untagFileGroup} from "../api";
import {Media, ThemeContext} from "../contexts/contexts";
@ -168,6 +168,11 @@ function DocPage() {
const navigate = useNavigate();
const {docFile, doc, fetchDoc} = useDoc(parseInt(fileGroupId));
const {theme} = useContext(ThemeContext);
const [searchParams] = useSearchParams();
// Deep-link params: `loc` = EPUB spine index, `page` = PDF page, `q` = search term.
const locParam = searchParams.get('loc');
const pageParam = searchParams.get('page');
const qParam = searchParams.get('q');
const title = docFile ? (docFile.title || docFile.name) : null;
useTitle(title);
@ -220,10 +225,22 @@ function DocPage() {
const isCbz = activeMimetype.includes('cbz') || activeMimetype.includes('cbr')
|| activeMimetype.includes('comicbook+zip') || activeMimetype.includes('comicbook-rar')
|| CBZ_EXTENSIONS.includes(activeExt);
const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
// Build EPUB viewer URL, propagating any deep-link params through to epub.html.
let viewerUrl = null;
if (isEpub) {
const epubParams = new URLSearchParams();
epubParams.set('url', downloadUrl);
if (locParam) epubParams.set('loc', locParam);
if (qParam) epubParams.set('q', qParam);
viewerUrl = `/epub/epub.html?${epubParams.toString()}`;
}
const openUrl = viewerUrl || (activePath ? `/media/${encodeMediaPath(activePath)}` : null);
const canEmbed = isEpub || isPdf;
const embedUrl = isEpub ? viewerUrl : (isPdf && activePath ? `/media/${encodeMediaPath(activePath)}` : null);
// PDFs use the browser's built-in viewer, which honors `#page=N` fragments.
const pdfSrc = isPdf && activePath
? `/media/${encodeMediaPath(activePath)}${pageParam ? `#page=${encodeURIComponent(pageParam)}` : ''}`
: null;
const embedUrl = isEpub ? viewerUrl : pdfSrc;
const handleDelete = async () => {
if (doc && doc.id) {

View file

@ -43,7 +43,7 @@ import {
useWROLMode
} from "../hooks/customHooks";
import {useFileWorkerStatus} from "../contexts/FileWorkerStatusContext";
import {Link, Route, Routes} from "react-router";
import {Link, Route, Routes, useSearchParams} from "react-router";
import {CardPlaceholder} from "./Placeholder";
import {ArchiveCard, ArchiveRowCells} from "./Archive";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
@ -67,15 +67,35 @@ import {Headlines} from "./Headline";
import {useSearch} from "./Search";
import {FILES_MEDIA_URI} from "./Vars";
// Split a ts_headline snippet on our sentinel markers and wrap each match in <b>.
// We use sentinels instead of ts_headline's default <b> tags so the snippet can be
// safely rendered as React children without dangerouslySetInnerHTML.
function renderHighlightedSnippet(snippet, s) {
const parts = snippet.split(/\[\[WROLPI_HL\]\]|\[\[\/WROLPI_HL\]\]/);
// Even indices are plain text, odd indices are highlighted matches.
return parts.map((part, i) => i % 2 === 1 ? <b key={i} {...s}>{part}</b> : <span key={i} {...s}>{part}</span>);
}
function EbookCard({file, sortField}) {
const {s, t} = useContext(ThemeContext);
const [searchParams] = useSearchParams();
const query = searchParams.get('q') || '';
const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
const isEpub = file['mimetype'] && file['mimetype'].startsWith('application/epub');
const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
// Link to doc detail page if this is a modeled doc.
const detailUrl = file.model === 'doc' ? `/docs/${file.id}` : null;
// Link to doc detail page if this is a modeled doc, augmenting with
// the section hint from search so the viewer opens at the right chapter/page.
const hint = file.section_hint;
let detailUrl = file.model === 'doc' ? `/docs/${file.id}` : null;
if (detailUrl && hint) {
const params = new URLSearchParams();
if (hint.kind === 'epub_spine') params.set('loc', String(hint.ordinal));
if (hint.kind === 'pdf_page') params.set('page', String(hint.ordinal));
if (query) params.set('q', query);
detailUrl = `${detailUrl}?${params.toString()}`;
}
const color = mimetypeColor(file.mimetype, file.primary_path);
const title = file.title || file.stem || file.name;
@ -98,6 +118,12 @@ function EbookCard({file, sortField}) {
? <p {...s}>{file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime, false) : null}</p>
: file.size && <p {...s}>{humanFileSize(file.size)}</p>
}
{hint && <p {...s} style={{fontSize: '0.85em', marginTop: '0.3em'}}>
<b {...s}>{hint.label}</b>
{hint.snippet && <span style={{...s, marginLeft: '0.4em', opacity: 0.8}}>
{renderHighlightedSnippet(hint.snippet, s)}
</span>}
</p>}
</CardMeta>
</CardContent>
</Card>

View file

@ -14,7 +14,7 @@ from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
from .extractors import extract_metadata
from .lib import get_or_create_subject_collection, get_or_create_author_collection, is_valid_author, split_authors, \
normalize_author, normalize_subject, split_subjects, is_valid_subject, discover_calibre_cover
from .models import Doc, DOC_MIMETYPES, COMIC_BOOK_SUFFIXES, EPUB_MIMETYPE, MOBI_MIMETYPE
from .models import Doc, DocSection, DOC_MIMETYPES, COMIC_BOOK_SUFFIXES, EPUB_MIMETYPE, MOBI_MIMETYPE
logger = logging.getLogger(__name__)
@ -118,10 +118,35 @@ def _model_doc(file_group: FileGroup, session: Session) -> Doc:
# Auto-create collections.
_auto_create_collections(session, file_group, metadata)
# Ensure the Doc has an id before inserting DocSection rows that reference it.
doc.flush()
# Replace any existing sections with the freshly-extracted ones.
_replace_doc_sections(session, doc, metadata.sections)
return doc
def _replace_doc_sections(session: Session, doc: Doc, sections):
"""Delete any DocSection rows for this Doc and insert the new ones.
Using a bulk delete + bulk insert keeps this O(N) without SQLAlchemy tracking
each old row. Callers must have flushed `doc` so it has an id.
"""
session.query(DocSection).filter(DocSection.doc_id == doc.id).delete(synchronize_session=False)
if not sections:
return
session.bulk_save_objects([
DocSection(
doc_id=doc.id,
kind=s.kind,
ordinal=s.ordinal,
label=s.label,
content=s.content,
) for s in sections
])
def _handle_doc_cover(doc: Doc, file_group: FileGroup, metadata):
"""Handle cover extraction and storage for a doc."""

View file

@ -4,6 +4,7 @@ import logging
import os
import pathlib
import zipfile
from typing import List
import pytz
@ -13,6 +14,10 @@ from wrolpi.files.indexers import Indexer
from wrolpi.files.models import FileGroup
from wrolpi.vars import FILE_MAX_PDF_SIZE, FILE_MAX_TEXT_SIZE
# Cap on a single section's stored text to prevent a pathological one-chapter-is-the-whole-book
# EPUB from producing a huge row.
SECTION_MAX_TEXT_SIZE = 200_000
try:
import ebooklib
from ebooklib import epub
@ -33,6 +38,19 @@ except ImportError:
logger = logging.getLogger(__name__)
@dataclasses.dataclass
class DocSection:
"""A searchable sub-range of a document (EPUB spine item or PDF page).
`ordinal` is 0-based for EPUB spine items and 1-based for PDF pages, to match the
conventions of the respective viewers.
"""
kind: str # 'epub_spine' or 'pdf_page'
ordinal: int
label: str
content: str
@dataclasses.dataclass
class DocMetadata:
title: str = None
@ -45,6 +63,7 @@ class DocMetadata:
description: str = None
cover_bytes: bytes = None
text: str = None
sections: List[DocSection] = dataclasses.field(default_factory=list)
def extract_metadata(file_group: FileGroup) -> DocMetadata:
@ -79,6 +98,46 @@ def _extract_text_from_html(html: str) -> str:
return soup.get_text()
def _build_epub_toc_label_map(toc) -> dict:
"""Walk an ebooklib TOC (nested list of Links and (Section, children) tuples)
and return a mapping of href (fragment-stripped) -> label.
The first label wins for any given href; children are walked so deeply-nested
entries still contribute. Anything without both an href and a title is skipped.
"""
href_to_label: dict = {}
def walk(entries):
for entry in entries or ():
if isinstance(entry, (list, tuple)):
walk(entry)
else:
href = getattr(entry, 'href', None)
title = getattr(entry, 'title', None)
if href and title:
href_to_label.setdefault(href.split('#')[0], title)
walk(toc)
return href_to_label
def _resolve_spine_labels(hrefs, href_to_label: dict) -> list:
"""Assign each spine item a label, carrying the most recently-matched TOC label
forward so mid-chapter spine items inherit their chapter's label.
`hrefs` is a list of (file_name, get_name) tuples in spine order. Items before
the first TOC match (e.g. cover, title page) get a generic `Section N` fallback.
"""
labels: list = []
current_label = None
for ordinal, (file_name, get_name) in enumerate(hrefs):
new_label = href_to_label.get(file_name) or href_to_label.get(get_name)
if new_label:
current_label = new_label
labels.append(current_label or f'Section {ordinal + 1}')
return labels
def extract_epub(path: pathlib.Path) -> DocMetadata:
"""Extract metadata from an EPUB file using ebooklib."""
if ebooklib is None or epub is None:
@ -107,10 +166,32 @@ def extract_epub(path: pathlib.Path) -> DocMetadata:
if 'subject' in value:
metadata.subject = metadata.subject or value['subject'][0][0]
# Extract text.
# Build a map of spine-item href -> chapter label from the TOC, if available.
href_to_label = {}
try:
href_to_label = _build_epub_toc_label_map(book.toc)
except Exception as e:
logger.warning(f'Failed to walk EPUB TOC for {path}', exc_info=e)
# Extract text per spine item, keeping a per-section record for deep-linking.
# Many EPUBs (especially Calibre-converted ones) split chapters across many spine
# items but only have a single TOC entry at the chapter start. Carry the most
# recently-matched TOC label forward so mid-chapter sections still get a useful
# label (e.g. "Mullein" rather than "Section 302").
text_parts = []
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
text_parts.append(_extract_text_from_html(doc.content))
spine_items = list(book.get_items_of_type(ebooklib.ITEM_DOCUMENT))
hrefs = [(d.file_name, d.get_name()) for d in spine_items]
labels = _resolve_spine_labels(hrefs, href_to_label)
for ordinal, (doc, label) in enumerate(zip(spine_items, labels)):
section_text = _extract_text_from_html(doc.content)
text_parts.append(section_text)
capped = truncate_object_bytes(section_text, SECTION_MAX_TEXT_SIZE)
metadata.sections.append(DocSection(
kind='epub_spine',
ordinal=ordinal,
label=label,
content=capped,
))
metadata.text = ''.join(text_parts)
# Extract cover.
@ -174,6 +255,14 @@ def extract_pdf(path: pathlib.Path) -> DocMetadata:
text = text.replace('\x00', '\uFFFD').strip()
if text:
words.append(text)
# Record a per-page section (1-based, matching the PDF viewer's
# `#page=N` fragment convention).
metadata.sections.append(DocSection(
kind='pdf_page',
ordinal=page.number + 1,
label=f'Page {page.number + 1}',
content=truncate_object_bytes(text, SECTION_MAX_TEXT_SIZE),
))
metadata.text = '\n'.join(
truncate_generator_bytes(iter(words), FILE_MAX_TEXT_SIZE))
except Exception as e:

View file

@ -4,7 +4,7 @@ import pathlib
import re
from typing import List, Optional
from sqlalchemy import asc, desc, func, nullslast
from sqlalchemy import asc, desc, func, nullslast, text
from sqlalchemy.orm import Session
from wrolpi.collections.models import Collection
@ -404,6 +404,50 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
query = query.order_by(desc(FileGroup.id))
file_groups = query.offset(offset).limit(limit).all()
fg_ids = [fg.id for fg in file_groups]
results = [fg.__json__() for fg in file_groups]
if search_str and fg_ids:
hints = _fetch_section_hints(session, fg_ids, search_str)
for r in results:
hint = hints.get(r['id'])
if hint:
r['section_hint'] = hint
return results, total
def _fetch_section_hints(session, file_group_ids, search_str):
"""For each matching Doc, return the best-ranking DocSection for `search_str`.
Returns a mapping of file_group_id -> {kind, ordinal, label, snippet}.
"""
from .models import Doc, DocSection
if not file_group_ids:
return {}
sql = text('''
SELECT DISTINCT ON (d.file_group_id)
d.file_group_id AS fg_id,
ds.kind AS kind,
ds.ordinal AS ordinal,
ds.label AS label,
ts_headline('english', ds.content, q,
'MaxWords=20,MinWords=5,ShortWord=3,StartSel=[[WROLPI_HL]],StopSel=[[/WROLPI_HL]]') AS snippet
FROM doc_section ds
JOIN doc d ON d.id = ds.doc_id,
websearch_to_tsquery('english', :q) q
WHERE d.file_group_id = ANY(:ids) AND ds.tsv @@ q
ORDER BY d.file_group_id, ts_rank(ds.tsv, q) DESC, ds.ordinal ASC
''')
rows = session.execute(sql, {'q': search_str, 'ids': list(file_group_ids)}).fetchall()
hints = {}
for row in rows:
hints[row['fg_id']] = {
'kind': row['kind'],
'ordinal': row['ordinal'],
'label': row['label'],
'snippet': row['snippet'],
}
return hints

View file

@ -1,10 +1,10 @@
import pathlib
from typing import Optional
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_
from sqlalchemy.orm import relationship, Session
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, Computed, or_
from sqlalchemy.orm import relationship, Session, deferred
from wrolpi.common import ModelHelper, Base
from wrolpi.common import ModelHelper, Base, tsvector
from wrolpi.db import get_db_session
from wrolpi.files.models import FileGroup
@ -57,6 +57,9 @@ class Doc(ModelHelper, Base):
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
file_group: FileGroup = relationship('FileGroup')
sections = relationship('DocSection', back_populates='doc',
cascade='all, delete-orphan',
passive_deletes=True)
def __repr__(self):
path = str(self.file_group.primary_path) if self.file_group else 'None'
@ -120,3 +123,37 @@ class Doc(ModelHelper, Base):
))
for fg in file_groups:
session.delete(fg)
# Section kinds.
SECTION_KIND_EPUB_SPINE = 'epub_spine'
SECTION_KIND_PDF_PAGE = 'pdf_page'
class DocSection(ModelHelper, Base):
"""A searchable sub-range of a Doc.
For EPUB files, one row per spine item with `ordinal` as the 0-based spine index.
For PDF files, one row per page with `ordinal` as the 1-based page number.
Each row maintains its own tsvector for per-section full-text search, which lets us
deep-link from a search result into a specific chapter or page.
"""
__tablename__ = 'doc_section'
__table_args__ = (
Index('doc_section_doc_id_idx', 'doc_id', 'ordinal'),
Index('doc_section_tsv_idx', 'tsv', postgresql_using='gin'),
)
id = Column(BigInteger, primary_key=True)
doc_id = Column(Integer, ForeignKey('doc.id', ondelete='CASCADE'), nullable=False)
kind = Column(String, nullable=False)
ordinal = Column(Integer, nullable=False)
label = Column(String)
content = Column(Text)
tsv = deferred(Column(tsvector, Computed(
"to_tsvector('english'::regconfig, COALESCE(content, ''))")))
doc = relationship('Doc', back_populates='sections')
def __repr__(self):
return f'<DocSection id={self.id} doc_id={self.doc_id} kind={self.kind} ordinal={self.ordinal}>'

View file

@ -6,7 +6,7 @@ import pytest
from PIL import Image
from modules.docs import doc_modeler
from modules.docs.models import Doc
from modules.docs.models import Doc, DocSection
from wrolpi.dates import now
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
@ -52,6 +52,53 @@ async def test_doc_modeler_pdf(async_client, test_session, test_directory, examp
assert doc.size # Has a size.
@pytest.mark.asyncio
async def test_doc_modeler_persists_epub_sections(async_client, test_session, test_directory,
example_epub, refresh_files):
"""After modeling, per-spine-item DocSection rows exist for an EPUB."""
await refresh_files()
doc: Doc = test_session.query(Doc).one()
sections = test_session.query(DocSection).filter_by(doc_id=doc.id) \
.order_by(DocSection.ordinal).all()
assert sections, 'EPUB modeler should produce DocSection rows'
assert all(s.kind == 'epub_spine' for s in sections)
assert [s.ordinal for s in sections] == list(range(len(sections)))
@pytest.mark.asyncio
async def test_doc_modeler_persists_pdf_sections(async_client, test_session, test_directory,
example_pdf, refresh_files):
"""After modeling, per-page DocSection rows exist for a PDF."""
await refresh_files()
doc: Doc = test_session.query(Doc).one()
sections = test_session.query(DocSection).filter_by(doc_id=doc.id) \
.order_by(DocSection.ordinal).all()
assert sections, 'PDF modeler should produce DocSection rows'
assert all(s.kind == 'pdf_page' for s in sections)
# 1-based ordinals, test PDF has 3 pages.
assert [s.ordinal for s in sections] == [1, 2, 3]
@pytest.mark.asyncio
async def test_doc_modeler_replaces_sections(async_client, test_session, test_directory,
example_epub, refresh_files):
"""Re-modeling a Doc replaces its DocSection rows (no duplicates)."""
from modules.docs import _model_doc
await refresh_files()
doc: Doc = test_session.query(Doc).one()
initial_count = test_session.query(DocSection).filter_by(doc_id=doc.id).count()
assert initial_count > 0
# Re-run modeling on the same file_group and confirm we don't accumulate rows.
_model_doc(doc.file_group, test_session)
test_session.commit()
final_count = test_session.query(DocSection).filter_by(doc_id=doc.id).count()
assert final_count == initial_count
@pytest.mark.asyncio
async def test_doc_do_model_sets_model(test_session, example_pdf):
"""Doc.do_model sets file_group.model so the FileGroup is recognized as a doc."""

View file

@ -15,6 +15,77 @@ async def test_extract_epub(test_session, test_directory, example_epub):
assert metadata.cover_bytes # Has a cover image.
@pytest.mark.asyncio
async def test_extract_epub_sections(test_session, test_directory, example_epub):
"""extract_epub produces one DocSection per spine item with ascending ordinals."""
metadata = extract_epub(example_epub)
assert metadata.sections, 'EPUB should produce at least one section'
assert all(s.kind == 'epub_spine' for s in metadata.sections)
# Ordinals should be 0..N-1 and strictly ascending.
ordinals = [s.ordinal for s in metadata.sections]
assert ordinals == list(range(len(metadata.sections)))
# Every section has a label.
assert all(s.label for s in metadata.sections)
def test_resolve_spine_labels_carries_forward_across_split_chapters():
"""When a chapter spans multiple spine items (common in Calibre conversions),
sections after the TOC-anchored one inherit the chapter label instead of
falling back to a generic 'Section N'.
This models real-world books like 'Mountain States Medicinal Plants' where each
plant's chapter is split across ~5 spine items but the TOC only anchors the first.
"""
from modules.docs.extractors import _resolve_spine_labels
href_to_label = {
'text/mullein_0.xhtml': 'Mullein',
'text/yarrow_0.xhtml': 'Yarrow',
}
# (file_name, get_name) tuples in spine order.
hrefs = [
('front.xhtml', 'front.xhtml'),
('text/mullein_0.xhtml', 'text/mullein_0.xhtml'),
('text/mullein_1.xhtml', 'text/mullein_1.xhtml'),
('text/mullein_2.xhtml', 'text/mullein_2.xhtml'),
('text/yarrow_0.xhtml', 'text/yarrow_0.xhtml'),
('text/yarrow_1.xhtml', 'text/yarrow_1.xhtml'),
]
labels = _resolve_spine_labels(hrefs, href_to_label)
# Front matter has no TOC anchor yet — falls back.
assert labels[0] == 'Section 1'
# mullein_0 is the TOC-anchored entry for 'Mullein'; the next two inherit it.
assert labels[1:4] == ['Mullein', 'Mullein', 'Mullein']
# yarrow_0 flips the carry-forward to the next chapter.
assert labels[4:6] == ['Yarrow', 'Yarrow']
def test_build_epub_toc_label_map_strips_fragments_and_walks_nested():
"""_build_epub_toc_label_map handles fragments and nested (Section, [children])
tuples, taking the first label that wins for a given href."""
from modules.docs.extractors import _build_epub_toc_label_map
class FakeLink:
def __init__(self, href, title):
self.href = href
self.title = title
toc = (
FakeLink('chap1.xhtml', 'Chapter One'),
# A nested (Section, [children]) tuple — children should be walked.
(FakeLink('chap2.xhtml', 'Part Two'), [
FakeLink('chap2.xhtml#sub-a', 'Sub A'),
FakeLink('chap3.xhtml', 'Chapter Three'),
]),
FakeLink('chap1.xhtml', 'Chapter One Duplicate'), # first wins
)
result = _build_epub_toc_label_map(toc)
assert result['chap1.xhtml'] == 'Chapter One'
assert result['chap2.xhtml'] == 'Part Two' # fragment stripped; first wins
assert result['chap3.xhtml'] == 'Chapter Three'
@pytest.mark.asyncio
async def test_extract_pdf(test_session, test_directory, example_pdf):
"""Metadata can be extracted from a PDF file."""
@ -36,6 +107,17 @@ async def test_extract_pdf_metadata(test_session, test_directory, example_pdf):
assert metadata.text
@pytest.mark.asyncio
async def test_extract_pdf_sections(test_session, test_directory, example_pdf):
"""extract_pdf produces one DocSection per page with 1-based ordinals."""
metadata = extract_pdf(example_pdf)
assert metadata.sections, 'PDF should produce at least one section'
assert all(s.kind == 'pdf_page' for s in metadata.sections)
# Pages are 1-indexed; the test PDF has 3 pages.
assert [s.ordinal for s in metadata.sections] == [1, 2, 3]
assert all(s.label == f'Page {s.ordinal}' for s in metadata.sections)
@pytest.mark.asyncio
async def test_extract_docx(test_session, test_directory, example_docx):
"""DOCX extractor extracts text content."""

View file

@ -2,7 +2,9 @@ import pytest
from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \
get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \
search_authors_by_name, search_subjects_by_name
search_authors_by_name, search_subjects_by_name, _search_docs
from modules.docs.models import Doc, DocSection
from wrolpi.files.models import FileGroup
def test_is_valid_author():
@ -248,3 +250,83 @@ async def test_search_subjects_by_name(test_session):
# No match.
results = await search_subjects_by_name(test_session, 'nothing')
assert len(results) == 0
def _make_doc_with_sections(test_session, test_directory, filename, mimetype, sections):
"""Build a FileGroup + Doc + DocSection rows for search tests.
`sections` is a list of (ordinal, label, content) tuples. The section contents
are also concatenated into the FileGroup's d_text so the top-level tsvector
search can match the doc in the first place.
"""
path = test_directory / filename
path.write_bytes(b'') # Placeholder; _search_docs doesn't read the file.
fg = FileGroup.from_paths(test_session, path)
fg.mimetype = mimetype
fg.model = 'doc'
fg.title = filename
fg.a_text = filename
fg.d_text = ' '.join(c for _, _, c in sections)
test_session.flush()
doc = Doc(file_group_id=fg.id, size=1)
test_session.add(doc)
test_session.flush()
kind = 'pdf_page' if mimetype == 'application/pdf' else 'epub_spine'
for ordinal, label, content in sections:
test_session.add(DocSection(doc_id=doc.id, kind=kind, ordinal=ordinal,
label=label, content=content))
test_session.commit()
return fg, doc
@pytest.mark.asyncio
async def test_search_docs_attaches_section_hint_epub(test_session, test_directory):
"""_search_docs returns a section_hint pointing at the matching spine item."""
fg, _ = _make_doc_with_sections(test_session, test_directory,
'alpha.epub', 'application/epub+zip', [
(0, 'Introduction', 'preface about gardening'),
(1, 'Chapter 1', 'a tale of mullen and his dog'),
(2, 'Chapter 2', 'unrelated content about pottery'),
])
results, total = _search_docs(search_str='mullen', mimetype='application/epub')
assert total == 1
assert results[0]['id'] == fg.id
hint = results[0]['section_hint']
assert hint['kind'] == 'epub_spine'
assert hint['ordinal'] == 1
assert hint['label'] == 'Chapter 1'
assert 'mullen' in hint['snippet'].lower()
# The snippet uses our sentinel markers rather than raw HTML.
assert '[[WROLPI_HL]]' in hint['snippet']
@pytest.mark.asyncio
async def test_search_docs_attaches_section_hint_pdf(test_session, test_directory):
"""_search_docs returns a section_hint pointing at the matching PDF page."""
fg, _ = _make_doc_with_sections(test_session, test_directory,
'beta.pdf', 'application/pdf', [
(1, 'Page 1', 'cover page text'),
(2, 'Page 2', 'nothing to see here'),
(3, 'Page 3', 'here is where mullen shows up'),
])
results, total = _search_docs(search_str='mullen', mimetype='application/pdf')
assert total == 1
assert results[0]['id'] == fg.id
hint = results[0]['section_hint']
assert hint['kind'] == 'pdf_page'
assert hint['ordinal'] == 3
assert hint['label'] == 'Page 3'
@pytest.mark.asyncio
async def test_search_docs_no_hint_when_no_query(test_session, test_directory):
"""Without a search query, results have no section_hint."""
fg, _ = _make_doc_with_sections(test_session, test_directory,
'gamma.pdf', 'application/pdf', [
(1, 'Page 1', 'whatever'),
])
results, _ = _search_docs(mimetype='application/pdf')
assert results and results[0]['id'] == fg.id
assert 'section_hint' not in results[0]

View file

@ -1052,6 +1052,19 @@ def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List
results[-1]['c_headline'] = extra['c_headline']
results[-1]['d_headline'] = extra['d_headline']
# When searching by text, attach a `section_hint` to any doc-modeled result so
# the UI can deep-link into the matching EPUB chapter or PDF page.
search_str = params.get('search_str') if isinstance(params, dict) else None
if search_str:
from modules.docs.lib import _fetch_section_hints
doc_ids = [r['id'] for r in results if r.get('model') == 'doc']
if doc_ids:
hints = _fetch_section_hints(session, doc_ids, search_str)
for r in results:
hint = hints.get(r['id'])
if hint:
r['section_hint'] = hint
return results, total

View file

@ -228,6 +228,34 @@ async def test_files_search(test_session, async_client, make_files_structure, as
await assert_files_search('nothing', [])
@pytest.mark.asyncio
async def test_files_search_attaches_doc_section_hint(async_client, test_session, test_directory,
example_epub, refresh_files):
"""The global /api/files/search endpoint attaches `section_hint` to doc-modeled
results so the UI can deep-link into the matching chapter. Regression test for a
bug where only the docs-specific endpoint (/api/docs/search) attached hints."""
await refresh_files()
body = dict(search_str='chapter')
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
epub_results = [r for r in response.json['file_groups']
if r.get('model') == 'doc'
and (r.get('mimetype') or '').startswith('application/epub')]
assert len(epub_results) == 1, f'Expected 1 EPUB result, got: {response.json}'
result = epub_results[0]
assert 'section_hint' in result, \
f'doc result has no section_hint on global search path: {result}'
hint = result['section_hint']
assert hint['kind'] == 'epub_spine'
# The test EPUB's "Chapter one" body is in spine item 1.
assert hint['ordinal'] == 1
assert hint['label']
# Snippet uses our sentinel markers, not raw HTML.
assert '[[WROLPI_HL]]' in hint['snippet']
@pytest.mark.asyncio
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory, refresh_files):
one, two = await tag_factory(), await tag_factory()