diff --git a/alembic/versions/d5e6f7a8b9c0_doc_section.py b/alembic/versions/d5e6f7a8b9c0_doc_section.py new file mode 100644 index 00000000..f85f2ecf --- /dev/null +++ b/alembic/versions/d5e6f7a8b9c0_doc_section.py @@ -0,0 +1,51 @@ +"""Create doc_section table for per-chapter/per-page full-text search. + +Enables deep-linking from search results into a specific EPUB chapter or PDF page, +and fixes a latent completeness bug in full-text search where FileGroup.d_text is +truncated to ~100KB (so only the first ~30 pages of a big PDF were searchable). + +Existing EPUB and PDF FileGroups are marked `indexed = FALSE` so that the regular +refresh pipeline re-runs their extractors and populates doc_section rows. No one-off +script is required. Note: the next refresh on a populated library will be heavier +than usual while every EPUB and PDF is re-extracted. + +Revision ID: d5e6f7a8b9c0 +Revises: c4d5e6f7a8b9 +Create Date: 2026-04-13 + +""" +from alembic import op + +revision = 'd5e6f7a8b9c0' +down_revision = 'c4d5e6f7a8b9' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute(''' + CREATE TABLE doc_section ( + id BIGSERIAL PRIMARY KEY, + doc_id INTEGER NOT NULL REFERENCES doc(id) ON DELETE CASCADE, + kind TEXT NOT NULL, + ordinal INTEGER NOT NULL, + label TEXT, + content TEXT, + tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english'::regconfig, COALESCE(content, ''))) STORED + ) + ''') + op.execute('CREATE INDEX doc_section_doc_id_idx ON doc_section (doc_id, ordinal)') + op.execute('CREATE INDEX doc_section_tsv_idx ON doc_section USING GIN (tsv)') + + # Force a re-index of all EPUB and PDF file groups so the refresh pipeline + # populates the new doc_section table. + op.execute(''' + UPDATE file_group + SET indexed = FALSE + WHERE mimetype LIKE 'application/epub%' + OR mimetype = 'application/pdf' + ''') + + +def downgrade(): + op.execute('DROP TABLE IF EXISTS doc_section') diff --git a/app/public/epub/epub.html b/app/public/epub/epub.html index b9c6011d..67945278 100644 --- a/app/public/epub/epub.html +++ b/app/public/epub/epub.html @@ -175,6 +175,9 @@ either expressed or implied, of the FreeBSD Project. --> let params = URLSearchParams && new URLSearchParams(document.location.search.substring(1)); let url = params && params.get("url") && decodeURIComponent(params.get("url")); let currentSectionIndex = (params && params.get("loc")) ? params.get("loc") : undefined; + // Optional search term passed in from a search-result deep link. If present we'll + // search the spine item at `loc` and jump to + highlight the first match. + let searchQuery = (params && params.get("q")) ? params.get("q") : null; let book = ePub(url); let rendition = book.renderTo("viewer", { @@ -185,6 +188,44 @@ either expressed or implied, of the FreeBSD Project. --> rendition.display(currentSectionIndex); + // If a search query was provided, try to locate it within the target spine item + // and scroll to / highlight the first match. Any failure is non-fatal — the `loc` + // jump above still lands the user at the right chapter. + if (searchQuery) { + book.ready.then(function () { + try { + let spine = book.spine; + let item; + if (currentSectionIndex !== undefined) { + let n = parseInt(currentSectionIndex, 10); + if (!isNaN(n)) { + item = spine.get(n); + } + } + if (!item && spine.spineItems && spine.spineItems.length) { + item = spine.spineItems[0]; + } + if (!item) return; + Promise.resolve(item.load(book.load.bind(book))).then(function () { + let results = item.find(searchQuery) || []; + if (results.length > 0) { + let hit = results[0]; + rendition.display(hit.cfi); + try { + rendition.annotations.highlight(hit.cfi); + } catch (e) { + console.warn("highlight failed", e); + } + } + }).finally(function () { + try { item.unload(); } catch (e) { /* ignore */ } + }); + } catch (e) { + console.warn("epub deep-link search failed", e); + } + }); + } + book.ready.then(() => { let next = document.getElementById("next"); diff --git a/app/src/components/Docs.js b/app/src/components/Docs.js index 2589b8e2..021ad121 100644 --- a/app/src/components/Docs.js +++ b/app/src/components/Docs.js @@ -1,5 +1,5 @@ import React, {useContext, useEffect, useState} from 'react'; -import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router"; +import {Link, Route, Routes, useLocation, useNavigate, useParams, useSearchParams} from "react-router"; import {Grid, Icon as SIcon, Loader, StatisticLabel, StatisticValue} from "semantic-ui-react"; import {deleteDocs, getDocStatistics, tagFileGroup, untagFileGroup} from "../api"; import {Media, ThemeContext} from "../contexts/contexts"; @@ -168,6 +168,11 @@ function DocPage() { const navigate = useNavigate(); const {docFile, doc, fetchDoc} = useDoc(parseInt(fileGroupId)); const {theme} = useContext(ThemeContext); + const [searchParams] = useSearchParams(); + // Deep-link params: `loc` = EPUB spine index, `page` = PDF page, `q` = search term. + const locParam = searchParams.get('loc'); + const pageParam = searchParams.get('page'); + const qParam = searchParams.get('q'); const title = docFile ? (docFile.title || docFile.name) : null; useTitle(title); @@ -220,10 +225,22 @@ function DocPage() { const isCbz = activeMimetype.includes('cbz') || activeMimetype.includes('cbr') || activeMimetype.includes('comicbook+zip') || activeMimetype.includes('comicbook-rar') || CBZ_EXTENSIONS.includes(activeExt); - const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null; + // Build EPUB viewer URL, propagating any deep-link params through to epub.html. + let viewerUrl = null; + if (isEpub) { + const epubParams = new URLSearchParams(); + epubParams.set('url', downloadUrl); + if (locParam) epubParams.set('loc', locParam); + if (qParam) epubParams.set('q', qParam); + viewerUrl = `/epub/epub.html?${epubParams.toString()}`; + } const openUrl = viewerUrl || (activePath ? `/media/${encodeMediaPath(activePath)}` : null); const canEmbed = isEpub || isPdf; - const embedUrl = isEpub ? viewerUrl : (isPdf && activePath ? `/media/${encodeMediaPath(activePath)}` : null); + // PDFs use the browser's built-in viewer, which honors `#page=N` fragments. + const pdfSrc = isPdf && activePath + ? `/media/${encodeMediaPath(activePath)}${pageParam ? `#page=${encodeURIComponent(pageParam)}` : ''}` + : null; + const embedUrl = isEpub ? viewerUrl : pdfSrc; const handleDelete = async () => { if (doc && doc.id) { diff --git a/app/src/components/Files.js b/app/src/components/Files.js index f1b7bdd1..ad83f959 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -43,7 +43,7 @@ import { useWROLMode } from "../hooks/customHooks"; import {useFileWorkerStatus} from "../contexts/FileWorkerStatusContext"; -import {Link, Route, Routes} from "react-router"; +import {Link, Route, Routes, useSearchParams} from "react-router"; import {CardPlaceholder} from "./Placeholder"; import {ArchiveCard, ArchiveRowCells} from "./Archive"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; @@ -67,15 +67,35 @@ import {Headlines} from "./Headline"; import {useSearch} from "./Search"; import {FILES_MEDIA_URI} from "./Vars"; +// Split a ts_headline snippet on our sentinel markers and wrap each match in . +// We use sentinels instead of ts_headline's default tags so the snippet can be +// safely rendered as React children without dangerouslySetInnerHTML. +function renderHighlightedSnippet(snippet, s) { + const parts = snippet.split(/\[\[WROLPI_HL\]\]|\[\[\/WROLPI_HL\]\]/); + // Even indices are plain text, odd indices are highlighted matches. + return parts.map((part, i) => i % 2 === 1 ? {part} : {part}); +} + function EbookCard({file, sortField}) { const {s, t} = useContext(ThemeContext); + const [searchParams] = useSearchParams(); + const query = searchParams.get('q') || ''; const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`; const isEpub = file['mimetype'] && file['mimetype'].startsWith('application/epub'); const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null; - // Link to doc detail page if this is a modeled doc. - const detailUrl = file.model === 'doc' ? `/docs/${file.id}` : null; + // Link to doc detail page if this is a modeled doc, augmenting with + // the section hint from search so the viewer opens at the right chapter/page. + const hint = file.section_hint; + let detailUrl = file.model === 'doc' ? `/docs/${file.id}` : null; + if (detailUrl && hint) { + const params = new URLSearchParams(); + if (hint.kind === 'epub_spine') params.set('loc', String(hint.ordinal)); + if (hint.kind === 'pdf_page') params.set('page', String(hint.ordinal)); + if (query) params.set('q', query); + detailUrl = `${detailUrl}?${params.toString()}`; + } const color = mimetypeColor(file.mimetype, file.primary_path); const title = file.title || file.stem || file.name; @@ -98,6 +118,12 @@ function EbookCard({file, sortField}) { ?

{file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime, false) : null}

: file.size &&

{humanFileSize(file.size)}

} + {hint &&

+ {hint.label} + {hint.snippet && + {renderHighlightedSnippet(hint.snippet, s)} + } +

} diff --git a/modules/docs/__init__.py b/modules/docs/__init__.py index 1b6f4f26..723b0eaf 100644 --- a/modules/docs/__init__.py +++ b/modules/docs/__init__.py @@ -14,7 +14,7 @@ from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE from .extractors import extract_metadata from .lib import get_or_create_subject_collection, get_or_create_author_collection, is_valid_author, split_authors, \ normalize_author, normalize_subject, split_subjects, is_valid_subject, discover_calibre_cover -from .models import Doc, DOC_MIMETYPES, COMIC_BOOK_SUFFIXES, EPUB_MIMETYPE, MOBI_MIMETYPE +from .models import Doc, DocSection, DOC_MIMETYPES, COMIC_BOOK_SUFFIXES, EPUB_MIMETYPE, MOBI_MIMETYPE logger = logging.getLogger(__name__) @@ -118,10 +118,35 @@ def _model_doc(file_group: FileGroup, session: Session) -> Doc: # Auto-create collections. _auto_create_collections(session, file_group, metadata) + # Ensure the Doc has an id before inserting DocSection rows that reference it. doc.flush() + + # Replace any existing sections with the freshly-extracted ones. + _replace_doc_sections(session, doc, metadata.sections) + return doc +def _replace_doc_sections(session: Session, doc: Doc, sections): + """Delete any DocSection rows for this Doc and insert the new ones. + + Using a bulk delete + bulk insert keeps this O(N) without SQLAlchemy tracking + each old row. Callers must have flushed `doc` so it has an id. + """ + session.query(DocSection).filter(DocSection.doc_id == doc.id).delete(synchronize_session=False) + if not sections: + return + session.bulk_save_objects([ + DocSection( + doc_id=doc.id, + kind=s.kind, + ordinal=s.ordinal, + label=s.label, + content=s.content, + ) for s in sections + ]) + + def _handle_doc_cover(doc: Doc, file_group: FileGroup, metadata): """Handle cover extraction and storage for a doc.""" diff --git a/modules/docs/extractors.py b/modules/docs/extractors.py index 46d0010b..f5074f2d 100644 --- a/modules/docs/extractors.py +++ b/modules/docs/extractors.py @@ -4,6 +4,7 @@ import logging import os import pathlib import zipfile +from typing import List import pytz @@ -13,6 +14,10 @@ from wrolpi.files.indexers import Indexer from wrolpi.files.models import FileGroup from wrolpi.vars import FILE_MAX_PDF_SIZE, FILE_MAX_TEXT_SIZE +# Cap on a single section's stored text to prevent a pathological one-chapter-is-the-whole-book +# EPUB from producing a huge row. +SECTION_MAX_TEXT_SIZE = 200_000 + try: import ebooklib from ebooklib import epub @@ -33,6 +38,19 @@ except ImportError: logger = logging.getLogger(__name__) +@dataclasses.dataclass +class DocSection: + """A searchable sub-range of a document (EPUB spine item or PDF page). + + `ordinal` is 0-based for EPUB spine items and 1-based for PDF pages, to match the + conventions of the respective viewers. + """ + kind: str # 'epub_spine' or 'pdf_page' + ordinal: int + label: str + content: str + + @dataclasses.dataclass class DocMetadata: title: str = None @@ -45,6 +63,7 @@ class DocMetadata: description: str = None cover_bytes: bytes = None text: str = None + sections: List[DocSection] = dataclasses.field(default_factory=list) def extract_metadata(file_group: FileGroup) -> DocMetadata: @@ -79,6 +98,46 @@ def _extract_text_from_html(html: str) -> str: return soup.get_text() +def _build_epub_toc_label_map(toc) -> dict: + """Walk an ebooklib TOC (nested list of Links and (Section, children) tuples) + and return a mapping of href (fragment-stripped) -> label. + + The first label wins for any given href; children are walked so deeply-nested + entries still contribute. Anything without both an href and a title is skipped. + """ + href_to_label: dict = {} + + def walk(entries): + for entry in entries or (): + if isinstance(entry, (list, tuple)): + walk(entry) + else: + href = getattr(entry, 'href', None) + title = getattr(entry, 'title', None) + if href and title: + href_to_label.setdefault(href.split('#')[0], title) + + walk(toc) + return href_to_label + + +def _resolve_spine_labels(hrefs, href_to_label: dict) -> list: + """Assign each spine item a label, carrying the most recently-matched TOC label + forward so mid-chapter spine items inherit their chapter's label. + + `hrefs` is a list of (file_name, get_name) tuples in spine order. Items before + the first TOC match (e.g. cover, title page) get a generic `Section N` fallback. + """ + labels: list = [] + current_label = None + for ordinal, (file_name, get_name) in enumerate(hrefs): + new_label = href_to_label.get(file_name) or href_to_label.get(get_name) + if new_label: + current_label = new_label + labels.append(current_label or f'Section {ordinal + 1}') + return labels + + def extract_epub(path: pathlib.Path) -> DocMetadata: """Extract metadata from an EPUB file using ebooklib.""" if ebooklib is None or epub is None: @@ -107,10 +166,32 @@ def extract_epub(path: pathlib.Path) -> DocMetadata: if 'subject' in value: metadata.subject = metadata.subject or value['subject'][0][0] - # Extract text. + # Build a map of spine-item href -> chapter label from the TOC, if available. + href_to_label = {} + try: + href_to_label = _build_epub_toc_label_map(book.toc) + except Exception as e: + logger.warning(f'Failed to walk EPUB TOC for {path}', exc_info=e) + + # Extract text per spine item, keeping a per-section record for deep-linking. + # Many EPUBs (especially Calibre-converted ones) split chapters across many spine + # items but only have a single TOC entry at the chapter start. Carry the most + # recently-matched TOC label forward so mid-chapter sections still get a useful + # label (e.g. "Mullein" rather than "Section 302"). text_parts = [] - for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT): - text_parts.append(_extract_text_from_html(doc.content)) + spine_items = list(book.get_items_of_type(ebooklib.ITEM_DOCUMENT)) + hrefs = [(d.file_name, d.get_name()) for d in spine_items] + labels = _resolve_spine_labels(hrefs, href_to_label) + for ordinal, (doc, label) in enumerate(zip(spine_items, labels)): + section_text = _extract_text_from_html(doc.content) + text_parts.append(section_text) + capped = truncate_object_bytes(section_text, SECTION_MAX_TEXT_SIZE) + metadata.sections.append(DocSection( + kind='epub_spine', + ordinal=ordinal, + label=label, + content=capped, + )) metadata.text = ''.join(text_parts) # Extract cover. @@ -174,6 +255,14 @@ def extract_pdf(path: pathlib.Path) -> DocMetadata: text = text.replace('\x00', '\uFFFD').strip() if text: words.append(text) + # Record a per-page section (1-based, matching the PDF viewer's + # `#page=N` fragment convention). + metadata.sections.append(DocSection( + kind='pdf_page', + ordinal=page.number + 1, + label=f'Page {page.number + 1}', + content=truncate_object_bytes(text, SECTION_MAX_TEXT_SIZE), + )) metadata.text = '\n'.join( truncate_generator_bytes(iter(words), FILE_MAX_TEXT_SIZE)) except Exception as e: diff --git a/modules/docs/lib.py b/modules/docs/lib.py index e8174919..e6a9be4d 100644 --- a/modules/docs/lib.py +++ b/modules/docs/lib.py @@ -4,7 +4,7 @@ import pathlib import re from typing import List, Optional -from sqlalchemy import asc, desc, func, nullslast +from sqlalchemy import asc, desc, func, nullslast, text from sqlalchemy.orm import Session from wrolpi.collections.models import Collection @@ -404,6 +404,50 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime query = query.order_by(desc(FileGroup.id)) file_groups = query.offset(offset).limit(limit).all() + fg_ids = [fg.id for fg in file_groups] results = [fg.__json__() for fg in file_groups] + if search_str and fg_ids: + hints = _fetch_section_hints(session, fg_ids, search_str) + for r in results: + hint = hints.get(r['id']) + if hint: + r['section_hint'] = hint + return results, total + + +def _fetch_section_hints(session, file_group_ids, search_str): + """For each matching Doc, return the best-ranking DocSection for `search_str`. + + Returns a mapping of file_group_id -> {kind, ordinal, label, snippet}. + """ + from .models import Doc, DocSection + + if not file_group_ids: + return {} + + sql = text(''' + SELECT DISTINCT ON (d.file_group_id) + d.file_group_id AS fg_id, + ds.kind AS kind, + ds.ordinal AS ordinal, + ds.label AS label, + ts_headline('english', ds.content, q, + 'MaxWords=20,MinWords=5,ShortWord=3,StartSel=[[WROLPI_HL]],StopSel=[[/WROLPI_HL]]') AS snippet + FROM doc_section ds + JOIN doc d ON d.id = ds.doc_id, + websearch_to_tsquery('english', :q) q + WHERE d.file_group_id = ANY(:ids) AND ds.tsv @@ q + ORDER BY d.file_group_id, ts_rank(ds.tsv, q) DESC, ds.ordinal ASC + ''') + rows = session.execute(sql, {'q': search_str, 'ids': list(file_group_ids)}).fetchall() + hints = {} + for row in rows: + hints[row['fg_id']] = { + 'kind': row['kind'], + 'ordinal': row['ordinal'], + 'label': row['label'], + 'snippet': row['snippet'], + } + return hints diff --git a/modules/docs/models.py b/modules/docs/models.py index faf79e11..da629063 100644 --- a/modules/docs/models.py +++ b/modules/docs/models.py @@ -1,10 +1,10 @@ import pathlib from typing import Optional -from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_ -from sqlalchemy.orm import relationship, Session +from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, Computed, or_ +from sqlalchemy.orm import relationship, Session, deferred -from wrolpi.common import ModelHelper, Base +from wrolpi.common import ModelHelper, Base, tsvector from wrolpi.db import get_db_session from wrolpi.files.models import FileGroup @@ -57,6 +57,9 @@ class Doc(ModelHelper, Base): file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True) file_group: FileGroup = relationship('FileGroup') + sections = relationship('DocSection', back_populates='doc', + cascade='all, delete-orphan', + passive_deletes=True) def __repr__(self): path = str(self.file_group.primary_path) if self.file_group else 'None' @@ -120,3 +123,37 @@ class Doc(ModelHelper, Base): )) for fg in file_groups: session.delete(fg) + + +# Section kinds. +SECTION_KIND_EPUB_SPINE = 'epub_spine' +SECTION_KIND_PDF_PAGE = 'pdf_page' + + +class DocSection(ModelHelper, Base): + """A searchable sub-range of a Doc. + + For EPUB files, one row per spine item with `ordinal` as the 0-based spine index. + For PDF files, one row per page with `ordinal` as the 1-based page number. + Each row maintains its own tsvector for per-section full-text search, which lets us + deep-link from a search result into a specific chapter or page. + """ + __tablename__ = 'doc_section' + __table_args__ = ( + Index('doc_section_doc_id_idx', 'doc_id', 'ordinal'), + Index('doc_section_tsv_idx', 'tsv', postgresql_using='gin'), + ) + + id = Column(BigInteger, primary_key=True) + doc_id = Column(Integer, ForeignKey('doc.id', ondelete='CASCADE'), nullable=False) + kind = Column(String, nullable=False) + ordinal = Column(Integer, nullable=False) + label = Column(String) + content = Column(Text) + tsv = deferred(Column(tsvector, Computed( + "to_tsvector('english'::regconfig, COALESCE(content, ''))"))) + + doc = relationship('Doc', back_populates='sections') + + def __repr__(self): + return f'' diff --git a/modules/docs/test/test_doc_modeler.py b/modules/docs/test/test_doc_modeler.py index ec84002e..b302a5c9 100644 --- a/modules/docs/test/test_doc_modeler.py +++ b/modules/docs/test/test_doc_modeler.py @@ -6,7 +6,7 @@ import pytest from PIL import Image from modules.docs import doc_modeler -from modules.docs.models import Doc +from modules.docs.models import Doc, DocSection from wrolpi.dates import now from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup @@ -52,6 +52,53 @@ async def test_doc_modeler_pdf(async_client, test_session, test_directory, examp assert doc.size # Has a size. +@pytest.mark.asyncio +async def test_doc_modeler_persists_epub_sections(async_client, test_session, test_directory, + example_epub, refresh_files): + """After modeling, per-spine-item DocSection rows exist for an EPUB.""" + await refresh_files() + + doc: Doc = test_session.query(Doc).one() + sections = test_session.query(DocSection).filter_by(doc_id=doc.id) \ + .order_by(DocSection.ordinal).all() + assert sections, 'EPUB modeler should produce DocSection rows' + assert all(s.kind == 'epub_spine' for s in sections) + assert [s.ordinal for s in sections] == list(range(len(sections))) + + +@pytest.mark.asyncio +async def test_doc_modeler_persists_pdf_sections(async_client, test_session, test_directory, + example_pdf, refresh_files): + """After modeling, per-page DocSection rows exist for a PDF.""" + await refresh_files() + + doc: Doc = test_session.query(Doc).one() + sections = test_session.query(DocSection).filter_by(doc_id=doc.id) \ + .order_by(DocSection.ordinal).all() + assert sections, 'PDF modeler should produce DocSection rows' + assert all(s.kind == 'pdf_page' for s in sections) + # 1-based ordinals, test PDF has 3 pages. + assert [s.ordinal for s in sections] == [1, 2, 3] + + +@pytest.mark.asyncio +async def test_doc_modeler_replaces_sections(async_client, test_session, test_directory, + example_epub, refresh_files): + """Re-modeling a Doc replaces its DocSection rows (no duplicates).""" + from modules.docs import _model_doc + + await refresh_files() + doc: Doc = test_session.query(Doc).one() + initial_count = test_session.query(DocSection).filter_by(doc_id=doc.id).count() + assert initial_count > 0 + + # Re-run modeling on the same file_group and confirm we don't accumulate rows. + _model_doc(doc.file_group, test_session) + test_session.commit() + final_count = test_session.query(DocSection).filter_by(doc_id=doc.id).count() + assert final_count == initial_count + + @pytest.mark.asyncio async def test_doc_do_model_sets_model(test_session, example_pdf): """Doc.do_model sets file_group.model so the FileGroup is recognized as a doc.""" diff --git a/modules/docs/test/test_extractors.py b/modules/docs/test/test_extractors.py index faf94f54..79995cbf 100644 --- a/modules/docs/test/test_extractors.py +++ b/modules/docs/test/test_extractors.py @@ -15,6 +15,77 @@ async def test_extract_epub(test_session, test_directory, example_epub): assert metadata.cover_bytes # Has a cover image. +@pytest.mark.asyncio +async def test_extract_epub_sections(test_session, test_directory, example_epub): + """extract_epub produces one DocSection per spine item with ascending ordinals.""" + metadata = extract_epub(example_epub) + assert metadata.sections, 'EPUB should produce at least one section' + assert all(s.kind == 'epub_spine' for s in metadata.sections) + # Ordinals should be 0..N-1 and strictly ascending. + ordinals = [s.ordinal for s in metadata.sections] + assert ordinals == list(range(len(metadata.sections))) + # Every section has a label. + assert all(s.label for s in metadata.sections) + + +def test_resolve_spine_labels_carries_forward_across_split_chapters(): + """When a chapter spans multiple spine items (common in Calibre conversions), + sections after the TOC-anchored one inherit the chapter label instead of + falling back to a generic 'Section N'. + + This models real-world books like 'Mountain States Medicinal Plants' where each + plant's chapter is split across ~5 spine items but the TOC only anchors the first. + """ + from modules.docs.extractors import _resolve_spine_labels + + href_to_label = { + 'text/mullein_0.xhtml': 'Mullein', + 'text/yarrow_0.xhtml': 'Yarrow', + } + # (file_name, get_name) tuples in spine order. + hrefs = [ + ('front.xhtml', 'front.xhtml'), + ('text/mullein_0.xhtml', 'text/mullein_0.xhtml'), + ('text/mullein_1.xhtml', 'text/mullein_1.xhtml'), + ('text/mullein_2.xhtml', 'text/mullein_2.xhtml'), + ('text/yarrow_0.xhtml', 'text/yarrow_0.xhtml'), + ('text/yarrow_1.xhtml', 'text/yarrow_1.xhtml'), + ] + labels = _resolve_spine_labels(hrefs, href_to_label) + + # Front matter has no TOC anchor yet — falls back. + assert labels[0] == 'Section 1' + # mullein_0 is the TOC-anchored entry for 'Mullein'; the next two inherit it. + assert labels[1:4] == ['Mullein', 'Mullein', 'Mullein'] + # yarrow_0 flips the carry-forward to the next chapter. + assert labels[4:6] == ['Yarrow', 'Yarrow'] + + +def test_build_epub_toc_label_map_strips_fragments_and_walks_nested(): + """_build_epub_toc_label_map handles fragments and nested (Section, [children]) + tuples, taking the first label that wins for a given href.""" + from modules.docs.extractors import _build_epub_toc_label_map + + class FakeLink: + def __init__(self, href, title): + self.href = href + self.title = title + + toc = ( + FakeLink('chap1.xhtml', 'Chapter One'), + # A nested (Section, [children]) tuple — children should be walked. + (FakeLink('chap2.xhtml', 'Part Two'), [ + FakeLink('chap2.xhtml#sub-a', 'Sub A'), + FakeLink('chap3.xhtml', 'Chapter Three'), + ]), + FakeLink('chap1.xhtml', 'Chapter One Duplicate'), # first wins + ) + result = _build_epub_toc_label_map(toc) + assert result['chap1.xhtml'] == 'Chapter One' + assert result['chap2.xhtml'] == 'Part Two' # fragment stripped; first wins + assert result['chap3.xhtml'] == 'Chapter Three' + + @pytest.mark.asyncio async def test_extract_pdf(test_session, test_directory, example_pdf): """Metadata can be extracted from a PDF file.""" @@ -36,6 +107,17 @@ async def test_extract_pdf_metadata(test_session, test_directory, example_pdf): assert metadata.text +@pytest.mark.asyncio +async def test_extract_pdf_sections(test_session, test_directory, example_pdf): + """extract_pdf produces one DocSection per page with 1-based ordinals.""" + metadata = extract_pdf(example_pdf) + assert metadata.sections, 'PDF should produce at least one section' + assert all(s.kind == 'pdf_page' for s in metadata.sections) + # Pages are 1-indexed; the test PDF has 3 pages. + assert [s.ordinal for s in metadata.sections] == [1, 2, 3] + assert all(s.label == f'Page {s.ordinal}' for s in metadata.sections) + + @pytest.mark.asyncio async def test_extract_docx(test_session, test_directory, example_docx): """DOCX extractor extracts text content.""" diff --git a/modules/docs/test/test_lib.py b/modules/docs/test/test_lib.py index 9752be64..c6dc661a 100644 --- a/modules/docs/test/test_lib.py +++ b/modules/docs/test/test_lib.py @@ -2,7 +2,9 @@ import pytest from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \ get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \ - search_authors_by_name, search_subjects_by_name + search_authors_by_name, search_subjects_by_name, _search_docs +from modules.docs.models import Doc, DocSection +from wrolpi.files.models import FileGroup def test_is_valid_author(): @@ -248,3 +250,83 @@ async def test_search_subjects_by_name(test_session): # No match. results = await search_subjects_by_name(test_session, 'nothing') assert len(results) == 0 + + +def _make_doc_with_sections(test_session, test_directory, filename, mimetype, sections): + """Build a FileGroup + Doc + DocSection rows for search tests. + + `sections` is a list of (ordinal, label, content) tuples. The section contents + are also concatenated into the FileGroup's d_text so the top-level tsvector + search can match the doc in the first place. + """ + path = test_directory / filename + path.write_bytes(b'') # Placeholder; _search_docs doesn't read the file. + fg = FileGroup.from_paths(test_session, path) + fg.mimetype = mimetype + fg.model = 'doc' + fg.title = filename + fg.a_text = filename + fg.d_text = ' '.join(c for _, _, c in sections) + test_session.flush() + doc = Doc(file_group_id=fg.id, size=1) + test_session.add(doc) + test_session.flush() + kind = 'pdf_page' if mimetype == 'application/pdf' else 'epub_spine' + for ordinal, label, content in sections: + test_session.add(DocSection(doc_id=doc.id, kind=kind, ordinal=ordinal, + label=label, content=content)) + test_session.commit() + return fg, doc + + +@pytest.mark.asyncio +async def test_search_docs_attaches_section_hint_epub(test_session, test_directory): + """_search_docs returns a section_hint pointing at the matching spine item.""" + fg, _ = _make_doc_with_sections(test_session, test_directory, + 'alpha.epub', 'application/epub+zip', [ + (0, 'Introduction', 'preface about gardening'), + (1, 'Chapter 1', 'a tale of mullen and his dog'), + (2, 'Chapter 2', 'unrelated content about pottery'), + ]) + + results, total = _search_docs(search_str='mullen', mimetype='application/epub') + assert total == 1 + assert results[0]['id'] == fg.id + hint = results[0]['section_hint'] + assert hint['kind'] == 'epub_spine' + assert hint['ordinal'] == 1 + assert hint['label'] == 'Chapter 1' + assert 'mullen' in hint['snippet'].lower() + # The snippet uses our sentinel markers rather than raw HTML. + assert '[[WROLPI_HL]]' in hint['snippet'] + + +@pytest.mark.asyncio +async def test_search_docs_attaches_section_hint_pdf(test_session, test_directory): + """_search_docs returns a section_hint pointing at the matching PDF page.""" + fg, _ = _make_doc_with_sections(test_session, test_directory, + 'beta.pdf', 'application/pdf', [ + (1, 'Page 1', 'cover page text'), + (2, 'Page 2', 'nothing to see here'), + (3, 'Page 3', 'here is where mullen shows up'), + ]) + + results, total = _search_docs(search_str='mullen', mimetype='application/pdf') + assert total == 1 + assert results[0]['id'] == fg.id + hint = results[0]['section_hint'] + assert hint['kind'] == 'pdf_page' + assert hint['ordinal'] == 3 + assert hint['label'] == 'Page 3' + + +@pytest.mark.asyncio +async def test_search_docs_no_hint_when_no_query(test_session, test_directory): + """Without a search query, results have no section_hint.""" + fg, _ = _make_doc_with_sections(test_session, test_directory, + 'gamma.pdf', 'application/pdf', [ + (1, 'Page 1', 'whatever'), + ]) + results, _ = _search_docs(mimetype='application/pdf') + assert results and results[0]['id'] == fg.id + assert 'section_hint' not in results[0] diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index b906eab8..8f573356 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1052,6 +1052,19 @@ def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List results[-1]['c_headline'] = extra['c_headline'] results[-1]['d_headline'] = extra['d_headline'] + # When searching by text, attach a `section_hint` to any doc-modeled result so + # the UI can deep-link into the matching EPUB chapter or PDF page. + search_str = params.get('search_str') if isinstance(params, dict) else None + if search_str: + from modules.docs.lib import _fetch_section_hints + doc_ids = [r['id'] for r in results if r.get('model') == 'doc'] + if doc_ids: + hints = _fetch_section_hints(session, doc_ids, search_str) + for r in results: + hint = hints.get(r['id']) + if hint: + r['section_hint'] = hint + return results, total diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 15596599..a0651f97 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -228,6 +228,34 @@ async def test_files_search(test_session, async_client, make_files_structure, as await assert_files_search('nothing', []) +@pytest.mark.asyncio +async def test_files_search_attaches_doc_section_hint(async_client, test_session, test_directory, + example_epub, refresh_files): + """The global /api/files/search endpoint attaches `section_hint` to doc-modeled + results so the UI can deep-link into the matching chapter. Regression test for a + bug where only the docs-specific endpoint (/api/docs/search) attached hints.""" + await refresh_files() + + body = dict(search_str='chapter') + request, response = await async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK + + epub_results = [r for r in response.json['file_groups'] + if r.get('model') == 'doc' + and (r.get('mimetype') or '').startswith('application/epub')] + assert len(epub_results) == 1, f'Expected 1 EPUB result, got: {response.json}' + result = epub_results[0] + assert 'section_hint' in result, \ + f'doc result has no section_hint on global search path: {result}' + hint = result['section_hint'] + assert hint['kind'] == 'epub_spine' + # The test EPUB's "Chapter one" body is in spine item 1. + assert hint['ordinal'] == 1 + assert hint['label'] + # Snippet uses our sentinel markers, not raw HTML. + assert '[[WROLPI_HL]]' in hint['snippet'] + + @pytest.mark.asyncio async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory, refresh_files): one, two = await tag_factory(), await tag_factory()