mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Linking to specific page or section of an EPUB/PDF when user searches.
This commit is contained in:
parent
673d851501
commit
8a75cf9057
13 changed files with 598 additions and 16 deletions
51
alembic/versions/d5e6f7a8b9c0_doc_section.py
Normal file
51
alembic/versions/d5e6f7a8b9c0_doc_section.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
"""Create doc_section table for per-chapter/per-page full-text search.
|
||||
|
||||
Enables deep-linking from search results into a specific EPUB chapter or PDF page,
|
||||
and fixes a latent completeness bug in full-text search where FileGroup.d_text is
|
||||
truncated to ~100KB (so only the first ~30 pages of a big PDF were searchable).
|
||||
|
||||
Existing EPUB and PDF FileGroups are marked `indexed = FALSE` so that the regular
|
||||
refresh pipeline re-runs their extractors and populates doc_section rows. No one-off
|
||||
script is required. Note: the next refresh on a populated library will be heavier
|
||||
than usual while every EPUB and PDF is re-extracted.
|
||||
|
||||
Revision ID: d5e6f7a8b9c0
|
||||
Revises: c4d5e6f7a8b9
|
||||
Create Date: 2026-04-13
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
revision = 'd5e6f7a8b9c0'
|
||||
down_revision = 'c4d5e6f7a8b9'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.execute('''
|
||||
CREATE TABLE doc_section (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
doc_id INTEGER NOT NULL REFERENCES doc(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL,
|
||||
ordinal INTEGER NOT NULL,
|
||||
label TEXT,
|
||||
content TEXT,
|
||||
tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english'::regconfig, COALESCE(content, ''))) STORED
|
||||
)
|
||||
''')
|
||||
op.execute('CREATE INDEX doc_section_doc_id_idx ON doc_section (doc_id, ordinal)')
|
||||
op.execute('CREATE INDEX doc_section_tsv_idx ON doc_section USING GIN (tsv)')
|
||||
|
||||
# Force a re-index of all EPUB and PDF file groups so the refresh pipeline
|
||||
# populates the new doc_section table.
|
||||
op.execute('''
|
||||
UPDATE file_group
|
||||
SET indexed = FALSE
|
||||
WHERE mimetype LIKE 'application/epub%'
|
||||
OR mimetype = 'application/pdf'
|
||||
''')
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.execute('DROP TABLE IF EXISTS doc_section')
|
||||
|
|
@ -175,6 +175,9 @@ either expressed or implied, of the FreeBSD Project. -->
|
|||
let params = URLSearchParams && new URLSearchParams(document.location.search.substring(1));
|
||||
let url = params && params.get("url") && decodeURIComponent(params.get("url"));
|
||||
let currentSectionIndex = (params && params.get("loc")) ? params.get("loc") : undefined;
|
||||
// Optional search term passed in from a search-result deep link. If present we'll
|
||||
// search the spine item at `loc` and jump to + highlight the first match.
|
||||
let searchQuery = (params && params.get("q")) ? params.get("q") : null;
|
||||
|
||||
let book = ePub(url);
|
||||
let rendition = book.renderTo("viewer", {
|
||||
|
|
@ -185,6 +188,44 @@ either expressed or implied, of the FreeBSD Project. -->
|
|||
|
||||
rendition.display(currentSectionIndex);
|
||||
|
||||
// If a search query was provided, try to locate it within the target spine item
|
||||
// and scroll to / highlight the first match. Any failure is non-fatal — the `loc`
|
||||
// jump above still lands the user at the right chapter.
|
||||
if (searchQuery) {
|
||||
book.ready.then(function () {
|
||||
try {
|
||||
let spine = book.spine;
|
||||
let item;
|
||||
if (currentSectionIndex !== undefined) {
|
||||
let n = parseInt(currentSectionIndex, 10);
|
||||
if (!isNaN(n)) {
|
||||
item = spine.get(n);
|
||||
}
|
||||
}
|
||||
if (!item && spine.spineItems && spine.spineItems.length) {
|
||||
item = spine.spineItems[0];
|
||||
}
|
||||
if (!item) return;
|
||||
Promise.resolve(item.load(book.load.bind(book))).then(function () {
|
||||
let results = item.find(searchQuery) || [];
|
||||
if (results.length > 0) {
|
||||
let hit = results[0];
|
||||
rendition.display(hit.cfi);
|
||||
try {
|
||||
rendition.annotations.highlight(hit.cfi);
|
||||
} catch (e) {
|
||||
console.warn("highlight failed", e);
|
||||
}
|
||||
}
|
||||
}).finally(function () {
|
||||
try { item.unload(); } catch (e) { /* ignore */ }
|
||||
});
|
||||
} catch (e) {
|
||||
console.warn("epub deep-link search failed", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
book.ready.then(() => {
|
||||
|
||||
let next = document.getElementById("next");
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import React, {useContext, useEffect, useState} from 'react';
|
||||
import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router";
|
||||
import {Link, Route, Routes, useLocation, useNavigate, useParams, useSearchParams} from "react-router";
|
||||
import {Grid, Icon as SIcon, Loader, StatisticLabel, StatisticValue} from "semantic-ui-react";
|
||||
import {deleteDocs, getDocStatistics, tagFileGroup, untagFileGroup} from "../api";
|
||||
import {Media, ThemeContext} from "../contexts/contexts";
|
||||
|
|
@ -168,6 +168,11 @@ function DocPage() {
|
|||
const navigate = useNavigate();
|
||||
const {docFile, doc, fetchDoc} = useDoc(parseInt(fileGroupId));
|
||||
const {theme} = useContext(ThemeContext);
|
||||
const [searchParams] = useSearchParams();
|
||||
// Deep-link params: `loc` = EPUB spine index, `page` = PDF page, `q` = search term.
|
||||
const locParam = searchParams.get('loc');
|
||||
const pageParam = searchParams.get('page');
|
||||
const qParam = searchParams.get('q');
|
||||
|
||||
const title = docFile ? (docFile.title || docFile.name) : null;
|
||||
useTitle(title);
|
||||
|
|
@ -220,10 +225,22 @@ function DocPage() {
|
|||
const isCbz = activeMimetype.includes('cbz') || activeMimetype.includes('cbr')
|
||||
|| activeMimetype.includes('comicbook+zip') || activeMimetype.includes('comicbook-rar')
|
||||
|| CBZ_EXTENSIONS.includes(activeExt);
|
||||
const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
|
||||
// Build EPUB viewer URL, propagating any deep-link params through to epub.html.
|
||||
let viewerUrl = null;
|
||||
if (isEpub) {
|
||||
const epubParams = new URLSearchParams();
|
||||
epubParams.set('url', downloadUrl);
|
||||
if (locParam) epubParams.set('loc', locParam);
|
||||
if (qParam) epubParams.set('q', qParam);
|
||||
viewerUrl = `/epub/epub.html?${epubParams.toString()}`;
|
||||
}
|
||||
const openUrl = viewerUrl || (activePath ? `/media/${encodeMediaPath(activePath)}` : null);
|
||||
const canEmbed = isEpub || isPdf;
|
||||
const embedUrl = isEpub ? viewerUrl : (isPdf && activePath ? `/media/${encodeMediaPath(activePath)}` : null);
|
||||
// PDFs use the browser's built-in viewer, which honors `#page=N` fragments.
|
||||
const pdfSrc = isPdf && activePath
|
||||
? `/media/${encodeMediaPath(activePath)}${pageParam ? `#page=${encodeURIComponent(pageParam)}` : ''}`
|
||||
: null;
|
||||
const embedUrl = isEpub ? viewerUrl : pdfSrc;
|
||||
|
||||
const handleDelete = async () => {
|
||||
if (doc && doc.id) {
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ import {
|
|||
useWROLMode
|
||||
} from "../hooks/customHooks";
|
||||
import {useFileWorkerStatus} from "../contexts/FileWorkerStatusContext";
|
||||
import {Link, Route, Routes} from "react-router";
|
||||
import {Link, Route, Routes, useSearchParams} from "react-router";
|
||||
import {CardPlaceholder} from "./Placeholder";
|
||||
import {ArchiveCard, ArchiveRowCells} from "./Archive";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
|
|
@ -67,15 +67,35 @@ import {Headlines} from "./Headline";
|
|||
import {useSearch} from "./Search";
|
||||
import {FILES_MEDIA_URI} from "./Vars";
|
||||
|
||||
// Split a ts_headline snippet on our sentinel markers and wrap each match in <b>.
|
||||
// We use sentinels instead of ts_headline's default <b> tags so the snippet can be
|
||||
// safely rendered as React children without dangerouslySetInnerHTML.
|
||||
function renderHighlightedSnippet(snippet, s) {
|
||||
const parts = snippet.split(/\[\[WROLPI_HL\]\]|\[\[\/WROLPI_HL\]\]/);
|
||||
// Even indices are plain text, odd indices are highlighted matches.
|
||||
return parts.map((part, i) => i % 2 === 1 ? <b key={i} {...s}>{part}</b> : <span key={i} {...s}>{part}</span>);
|
||||
}
|
||||
|
||||
function EbookCard({file, sortField}) {
|
||||
const {s, t} = useContext(ThemeContext);
|
||||
const [searchParams] = useSearchParams();
|
||||
const query = searchParams.get('q') || '';
|
||||
|
||||
const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
|
||||
const isEpub = file['mimetype'] && file['mimetype'].startsWith('application/epub');
|
||||
const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
|
||||
|
||||
// Link to doc detail page if this is a modeled doc.
|
||||
const detailUrl = file.model === 'doc' ? `/docs/${file.id}` : null;
|
||||
// Link to doc detail page if this is a modeled doc, augmenting with
|
||||
// the section hint from search so the viewer opens at the right chapter/page.
|
||||
const hint = file.section_hint;
|
||||
let detailUrl = file.model === 'doc' ? `/docs/${file.id}` : null;
|
||||
if (detailUrl && hint) {
|
||||
const params = new URLSearchParams();
|
||||
if (hint.kind === 'epub_spine') params.set('loc', String(hint.ordinal));
|
||||
if (hint.kind === 'pdf_page') params.set('page', String(hint.ordinal));
|
||||
if (query) params.set('q', query);
|
||||
detailUrl = `${detailUrl}?${params.toString()}`;
|
||||
}
|
||||
|
||||
const color = mimetypeColor(file.mimetype, file.primary_path);
|
||||
const title = file.title || file.stem || file.name;
|
||||
|
|
@ -98,6 +118,12 @@ function EbookCard({file, sortField}) {
|
|||
? <p {...s}>{file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime, false) : null}</p>
|
||||
: file.size && <p {...s}>{humanFileSize(file.size)}</p>
|
||||
}
|
||||
{hint && <p {...s} style={{fontSize: '0.85em', marginTop: '0.3em'}}>
|
||||
<b {...s}>{hint.label}</b>
|
||||
{hint.snippet && <span style={{...s, marginLeft: '0.4em', opacity: 0.8}}>
|
||||
{renderHighlightedSnippet(hint.snippet, s)}
|
||||
</span>}
|
||||
</p>}
|
||||
</CardMeta>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
|
|||
from .extractors import extract_metadata
|
||||
from .lib import get_or_create_subject_collection, get_or_create_author_collection, is_valid_author, split_authors, \
|
||||
normalize_author, normalize_subject, split_subjects, is_valid_subject, discover_calibre_cover
|
||||
from .models import Doc, DOC_MIMETYPES, COMIC_BOOK_SUFFIXES, EPUB_MIMETYPE, MOBI_MIMETYPE
|
||||
from .models import Doc, DocSection, DOC_MIMETYPES, COMIC_BOOK_SUFFIXES, EPUB_MIMETYPE, MOBI_MIMETYPE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -118,10 +118,35 @@ def _model_doc(file_group: FileGroup, session: Session) -> Doc:
|
|||
# Auto-create collections.
|
||||
_auto_create_collections(session, file_group, metadata)
|
||||
|
||||
# Ensure the Doc has an id before inserting DocSection rows that reference it.
|
||||
doc.flush()
|
||||
|
||||
# Replace any existing sections with the freshly-extracted ones.
|
||||
_replace_doc_sections(session, doc, metadata.sections)
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def _replace_doc_sections(session: Session, doc: Doc, sections):
|
||||
"""Delete any DocSection rows for this Doc and insert the new ones.
|
||||
|
||||
Using a bulk delete + bulk insert keeps this O(N) without SQLAlchemy tracking
|
||||
each old row. Callers must have flushed `doc` so it has an id.
|
||||
"""
|
||||
session.query(DocSection).filter(DocSection.doc_id == doc.id).delete(synchronize_session=False)
|
||||
if not sections:
|
||||
return
|
||||
session.bulk_save_objects([
|
||||
DocSection(
|
||||
doc_id=doc.id,
|
||||
kind=s.kind,
|
||||
ordinal=s.ordinal,
|
||||
label=s.label,
|
||||
content=s.content,
|
||||
) for s in sections
|
||||
])
|
||||
|
||||
|
||||
def _handle_doc_cover(doc: Doc, file_group: FileGroup, metadata):
|
||||
"""Handle cover extraction and storage for a doc."""
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import logging
|
|||
import os
|
||||
import pathlib
|
||||
import zipfile
|
||||
from typing import List
|
||||
|
||||
import pytz
|
||||
|
||||
|
|
@ -13,6 +14,10 @@ from wrolpi.files.indexers import Indexer
|
|||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import FILE_MAX_PDF_SIZE, FILE_MAX_TEXT_SIZE
|
||||
|
||||
# Cap on a single section's stored text to prevent a pathological one-chapter-is-the-whole-book
|
||||
# EPUB from producing a huge row.
|
||||
SECTION_MAX_TEXT_SIZE = 200_000
|
||||
|
||||
try:
|
||||
import ebooklib
|
||||
from ebooklib import epub
|
||||
|
|
@ -33,6 +38,19 @@ except ImportError:
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocSection:
|
||||
"""A searchable sub-range of a document (EPUB spine item or PDF page).
|
||||
|
||||
`ordinal` is 0-based for EPUB spine items and 1-based for PDF pages, to match the
|
||||
conventions of the respective viewers.
|
||||
"""
|
||||
kind: str # 'epub_spine' or 'pdf_page'
|
||||
ordinal: int
|
||||
label: str
|
||||
content: str
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocMetadata:
|
||||
title: str = None
|
||||
|
|
@ -45,6 +63,7 @@ class DocMetadata:
|
|||
description: str = None
|
||||
cover_bytes: bytes = None
|
||||
text: str = None
|
||||
sections: List[DocSection] = dataclasses.field(default_factory=list)
|
||||
|
||||
|
||||
def extract_metadata(file_group: FileGroup) -> DocMetadata:
|
||||
|
|
@ -79,6 +98,46 @@ def _extract_text_from_html(html: str) -> str:
|
|||
return soup.get_text()
|
||||
|
||||
|
||||
def _build_epub_toc_label_map(toc) -> dict:
|
||||
"""Walk an ebooklib TOC (nested list of Links and (Section, children) tuples)
|
||||
and return a mapping of href (fragment-stripped) -> label.
|
||||
|
||||
The first label wins for any given href; children are walked so deeply-nested
|
||||
entries still contribute. Anything without both an href and a title is skipped.
|
||||
"""
|
||||
href_to_label: dict = {}
|
||||
|
||||
def walk(entries):
|
||||
for entry in entries or ():
|
||||
if isinstance(entry, (list, tuple)):
|
||||
walk(entry)
|
||||
else:
|
||||
href = getattr(entry, 'href', None)
|
||||
title = getattr(entry, 'title', None)
|
||||
if href and title:
|
||||
href_to_label.setdefault(href.split('#')[0], title)
|
||||
|
||||
walk(toc)
|
||||
return href_to_label
|
||||
|
||||
|
||||
def _resolve_spine_labels(hrefs, href_to_label: dict) -> list:
|
||||
"""Assign each spine item a label, carrying the most recently-matched TOC label
|
||||
forward so mid-chapter spine items inherit their chapter's label.
|
||||
|
||||
`hrefs` is a list of (file_name, get_name) tuples in spine order. Items before
|
||||
the first TOC match (e.g. cover, title page) get a generic `Section N` fallback.
|
||||
"""
|
||||
labels: list = []
|
||||
current_label = None
|
||||
for ordinal, (file_name, get_name) in enumerate(hrefs):
|
||||
new_label = href_to_label.get(file_name) or href_to_label.get(get_name)
|
||||
if new_label:
|
||||
current_label = new_label
|
||||
labels.append(current_label or f'Section {ordinal + 1}')
|
||||
return labels
|
||||
|
||||
|
||||
def extract_epub(path: pathlib.Path) -> DocMetadata:
|
||||
"""Extract metadata from an EPUB file using ebooklib."""
|
||||
if ebooklib is None or epub is None:
|
||||
|
|
@ -107,10 +166,32 @@ def extract_epub(path: pathlib.Path) -> DocMetadata:
|
|||
if 'subject' in value:
|
||||
metadata.subject = metadata.subject or value['subject'][0][0]
|
||||
|
||||
# Extract text.
|
||||
# Build a map of spine-item href -> chapter label from the TOC, if available.
|
||||
href_to_label = {}
|
||||
try:
|
||||
href_to_label = _build_epub_toc_label_map(book.toc)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to walk EPUB TOC for {path}', exc_info=e)
|
||||
|
||||
# Extract text per spine item, keeping a per-section record for deep-linking.
|
||||
# Many EPUBs (especially Calibre-converted ones) split chapters across many spine
|
||||
# items but only have a single TOC entry at the chapter start. Carry the most
|
||||
# recently-matched TOC label forward so mid-chapter sections still get a useful
|
||||
# label (e.g. "Mullein" rather than "Section 302").
|
||||
text_parts = []
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||||
text_parts.append(_extract_text_from_html(doc.content))
|
||||
spine_items = list(book.get_items_of_type(ebooklib.ITEM_DOCUMENT))
|
||||
hrefs = [(d.file_name, d.get_name()) for d in spine_items]
|
||||
labels = _resolve_spine_labels(hrefs, href_to_label)
|
||||
for ordinal, (doc, label) in enumerate(zip(spine_items, labels)):
|
||||
section_text = _extract_text_from_html(doc.content)
|
||||
text_parts.append(section_text)
|
||||
capped = truncate_object_bytes(section_text, SECTION_MAX_TEXT_SIZE)
|
||||
metadata.sections.append(DocSection(
|
||||
kind='epub_spine',
|
||||
ordinal=ordinal,
|
||||
label=label,
|
||||
content=capped,
|
||||
))
|
||||
metadata.text = ''.join(text_parts)
|
||||
|
||||
# Extract cover.
|
||||
|
|
@ -174,6 +255,14 @@ def extract_pdf(path: pathlib.Path) -> DocMetadata:
|
|||
text = text.replace('\x00', '\uFFFD').strip()
|
||||
if text:
|
||||
words.append(text)
|
||||
# Record a per-page section (1-based, matching the PDF viewer's
|
||||
# `#page=N` fragment convention).
|
||||
metadata.sections.append(DocSection(
|
||||
kind='pdf_page',
|
||||
ordinal=page.number + 1,
|
||||
label=f'Page {page.number + 1}',
|
||||
content=truncate_object_bytes(text, SECTION_MAX_TEXT_SIZE),
|
||||
))
|
||||
metadata.text = '\n'.join(
|
||||
truncate_generator_bytes(iter(words), FILE_MAX_TEXT_SIZE))
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import pathlib
|
|||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import asc, desc, func, nullslast
|
||||
from sqlalchemy import asc, desc, func, nullslast, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.collections.models import Collection
|
||||
|
|
@ -404,6 +404,50 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
|
|||
query = query.order_by(desc(FileGroup.id))
|
||||
|
||||
file_groups = query.offset(offset).limit(limit).all()
|
||||
fg_ids = [fg.id for fg in file_groups]
|
||||
results = [fg.__json__() for fg in file_groups]
|
||||
|
||||
if search_str and fg_ids:
|
||||
hints = _fetch_section_hints(session, fg_ids, search_str)
|
||||
for r in results:
|
||||
hint = hints.get(r['id'])
|
||||
if hint:
|
||||
r['section_hint'] = hint
|
||||
|
||||
return results, total
|
||||
|
||||
|
||||
def _fetch_section_hints(session, file_group_ids, search_str):
|
||||
"""For each matching Doc, return the best-ranking DocSection for `search_str`.
|
||||
|
||||
Returns a mapping of file_group_id -> {kind, ordinal, label, snippet}.
|
||||
"""
|
||||
from .models import Doc, DocSection
|
||||
|
||||
if not file_group_ids:
|
||||
return {}
|
||||
|
||||
sql = text('''
|
||||
SELECT DISTINCT ON (d.file_group_id)
|
||||
d.file_group_id AS fg_id,
|
||||
ds.kind AS kind,
|
||||
ds.ordinal AS ordinal,
|
||||
ds.label AS label,
|
||||
ts_headline('english', ds.content, q,
|
||||
'MaxWords=20,MinWords=5,ShortWord=3,StartSel=[[WROLPI_HL]],StopSel=[[/WROLPI_HL]]') AS snippet
|
||||
FROM doc_section ds
|
||||
JOIN doc d ON d.id = ds.doc_id,
|
||||
websearch_to_tsquery('english', :q) q
|
||||
WHERE d.file_group_id = ANY(:ids) AND ds.tsv @@ q
|
||||
ORDER BY d.file_group_id, ts_rank(ds.tsv, q) DESC, ds.ordinal ASC
|
||||
''')
|
||||
rows = session.execute(sql, {'q': search_str, 'ids': list(file_group_ids)}).fetchall()
|
||||
hints = {}
|
||||
for row in rows:
|
||||
hints[row['fg_id']] = {
|
||||
'kind': row['kind'],
|
||||
'ordinal': row['ordinal'],
|
||||
'label': row['label'],
|
||||
'snippet': row['snippet'],
|
||||
}
|
||||
return hints
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import pathlib
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, Computed, or_
|
||||
from sqlalchemy.orm import relationship, Session, deferred
|
||||
|
||||
from wrolpi.common import ModelHelper, Base
|
||||
from wrolpi.common import ModelHelper, Base, tsvector
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
|
@ -57,6 +57,9 @@ class Doc(ModelHelper, Base):
|
|||
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
sections = relationship('DocSection', back_populates='doc',
|
||||
cascade='all, delete-orphan',
|
||||
passive_deletes=True)
|
||||
|
||||
def __repr__(self):
|
||||
path = str(self.file_group.primary_path) if self.file_group else 'None'
|
||||
|
|
@ -120,3 +123,37 @@ class Doc(ModelHelper, Base):
|
|||
))
|
||||
for fg in file_groups:
|
||||
session.delete(fg)
|
||||
|
||||
|
||||
# Section kinds.
|
||||
SECTION_KIND_EPUB_SPINE = 'epub_spine'
|
||||
SECTION_KIND_PDF_PAGE = 'pdf_page'
|
||||
|
||||
|
||||
class DocSection(ModelHelper, Base):
|
||||
"""A searchable sub-range of a Doc.
|
||||
|
||||
For EPUB files, one row per spine item with `ordinal` as the 0-based spine index.
|
||||
For PDF files, one row per page with `ordinal` as the 1-based page number.
|
||||
Each row maintains its own tsvector for per-section full-text search, which lets us
|
||||
deep-link from a search result into a specific chapter or page.
|
||||
"""
|
||||
__tablename__ = 'doc_section'
|
||||
__table_args__ = (
|
||||
Index('doc_section_doc_id_idx', 'doc_id', 'ordinal'),
|
||||
Index('doc_section_tsv_idx', 'tsv', postgresql_using='gin'),
|
||||
)
|
||||
|
||||
id = Column(BigInteger, primary_key=True)
|
||||
doc_id = Column(Integer, ForeignKey('doc.id', ondelete='CASCADE'), nullable=False)
|
||||
kind = Column(String, nullable=False)
|
||||
ordinal = Column(Integer, nullable=False)
|
||||
label = Column(String)
|
||||
content = Column(Text)
|
||||
tsv = deferred(Column(tsvector, Computed(
|
||||
"to_tsvector('english'::regconfig, COALESCE(content, ''))")))
|
||||
|
||||
doc = relationship('Doc', back_populates='sections')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<DocSection id={self.id} doc_id={self.doc_id} kind={self.kind} ordinal={self.ordinal}>'
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import pytest
|
|||
from PIL import Image
|
||||
|
||||
from modules.docs import doc_modeler
|
||||
from modules.docs.models import Doc
|
||||
from modules.docs.models import Doc, DocSection
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -52,6 +52,53 @@ async def test_doc_modeler_pdf(async_client, test_session, test_directory, examp
|
|||
assert doc.size # Has a size.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_persists_epub_sections(async_client, test_session, test_directory,
|
||||
example_epub, refresh_files):
|
||||
"""After modeling, per-spine-item DocSection rows exist for an EPUB."""
|
||||
await refresh_files()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
sections = test_session.query(DocSection).filter_by(doc_id=doc.id) \
|
||||
.order_by(DocSection.ordinal).all()
|
||||
assert sections, 'EPUB modeler should produce DocSection rows'
|
||||
assert all(s.kind == 'epub_spine' for s in sections)
|
||||
assert [s.ordinal for s in sections] == list(range(len(sections)))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_persists_pdf_sections(async_client, test_session, test_directory,
|
||||
example_pdf, refresh_files):
|
||||
"""After modeling, per-page DocSection rows exist for a PDF."""
|
||||
await refresh_files()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
sections = test_session.query(DocSection).filter_by(doc_id=doc.id) \
|
||||
.order_by(DocSection.ordinal).all()
|
||||
assert sections, 'PDF modeler should produce DocSection rows'
|
||||
assert all(s.kind == 'pdf_page' for s in sections)
|
||||
# 1-based ordinals, test PDF has 3 pages.
|
||||
assert [s.ordinal for s in sections] == [1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_replaces_sections(async_client, test_session, test_directory,
|
||||
example_epub, refresh_files):
|
||||
"""Re-modeling a Doc replaces its DocSection rows (no duplicates)."""
|
||||
from modules.docs import _model_doc
|
||||
|
||||
await refresh_files()
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
initial_count = test_session.query(DocSection).filter_by(doc_id=doc.id).count()
|
||||
assert initial_count > 0
|
||||
|
||||
# Re-run modeling on the same file_group and confirm we don't accumulate rows.
|
||||
_model_doc(doc.file_group, test_session)
|
||||
test_session.commit()
|
||||
final_count = test_session.query(DocSection).filter_by(doc_id=doc.id).count()
|
||||
assert final_count == initial_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_do_model_sets_model(test_session, example_pdf):
|
||||
"""Doc.do_model sets file_group.model so the FileGroup is recognized as a doc."""
|
||||
|
|
|
|||
|
|
@ -15,6 +15,77 @@ async def test_extract_epub(test_session, test_directory, example_epub):
|
|||
assert metadata.cover_bytes # Has a cover image.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_epub_sections(test_session, test_directory, example_epub):
|
||||
"""extract_epub produces one DocSection per spine item with ascending ordinals."""
|
||||
metadata = extract_epub(example_epub)
|
||||
assert metadata.sections, 'EPUB should produce at least one section'
|
||||
assert all(s.kind == 'epub_spine' for s in metadata.sections)
|
||||
# Ordinals should be 0..N-1 and strictly ascending.
|
||||
ordinals = [s.ordinal for s in metadata.sections]
|
||||
assert ordinals == list(range(len(metadata.sections)))
|
||||
# Every section has a label.
|
||||
assert all(s.label for s in metadata.sections)
|
||||
|
||||
|
||||
def test_resolve_spine_labels_carries_forward_across_split_chapters():
|
||||
"""When a chapter spans multiple spine items (common in Calibre conversions),
|
||||
sections after the TOC-anchored one inherit the chapter label instead of
|
||||
falling back to a generic 'Section N'.
|
||||
|
||||
This models real-world books like 'Mountain States Medicinal Plants' where each
|
||||
plant's chapter is split across ~5 spine items but the TOC only anchors the first.
|
||||
"""
|
||||
from modules.docs.extractors import _resolve_spine_labels
|
||||
|
||||
href_to_label = {
|
||||
'text/mullein_0.xhtml': 'Mullein',
|
||||
'text/yarrow_0.xhtml': 'Yarrow',
|
||||
}
|
||||
# (file_name, get_name) tuples in spine order.
|
||||
hrefs = [
|
||||
('front.xhtml', 'front.xhtml'),
|
||||
('text/mullein_0.xhtml', 'text/mullein_0.xhtml'),
|
||||
('text/mullein_1.xhtml', 'text/mullein_1.xhtml'),
|
||||
('text/mullein_2.xhtml', 'text/mullein_2.xhtml'),
|
||||
('text/yarrow_0.xhtml', 'text/yarrow_0.xhtml'),
|
||||
('text/yarrow_1.xhtml', 'text/yarrow_1.xhtml'),
|
||||
]
|
||||
labels = _resolve_spine_labels(hrefs, href_to_label)
|
||||
|
||||
# Front matter has no TOC anchor yet — falls back.
|
||||
assert labels[0] == 'Section 1'
|
||||
# mullein_0 is the TOC-anchored entry for 'Mullein'; the next two inherit it.
|
||||
assert labels[1:4] == ['Mullein', 'Mullein', 'Mullein']
|
||||
# yarrow_0 flips the carry-forward to the next chapter.
|
||||
assert labels[4:6] == ['Yarrow', 'Yarrow']
|
||||
|
||||
|
||||
def test_build_epub_toc_label_map_strips_fragments_and_walks_nested():
|
||||
"""_build_epub_toc_label_map handles fragments and nested (Section, [children])
|
||||
tuples, taking the first label that wins for a given href."""
|
||||
from modules.docs.extractors import _build_epub_toc_label_map
|
||||
|
||||
class FakeLink:
|
||||
def __init__(self, href, title):
|
||||
self.href = href
|
||||
self.title = title
|
||||
|
||||
toc = (
|
||||
FakeLink('chap1.xhtml', 'Chapter One'),
|
||||
# A nested (Section, [children]) tuple — children should be walked.
|
||||
(FakeLink('chap2.xhtml', 'Part Two'), [
|
||||
FakeLink('chap2.xhtml#sub-a', 'Sub A'),
|
||||
FakeLink('chap3.xhtml', 'Chapter Three'),
|
||||
]),
|
||||
FakeLink('chap1.xhtml', 'Chapter One Duplicate'), # first wins
|
||||
)
|
||||
result = _build_epub_toc_label_map(toc)
|
||||
assert result['chap1.xhtml'] == 'Chapter One'
|
||||
assert result['chap2.xhtml'] == 'Part Two' # fragment stripped; first wins
|
||||
assert result['chap3.xhtml'] == 'Chapter Three'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_pdf(test_session, test_directory, example_pdf):
|
||||
"""Metadata can be extracted from a PDF file."""
|
||||
|
|
@ -36,6 +107,17 @@ async def test_extract_pdf_metadata(test_session, test_directory, example_pdf):
|
|||
assert metadata.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_pdf_sections(test_session, test_directory, example_pdf):
|
||||
"""extract_pdf produces one DocSection per page with 1-based ordinals."""
|
||||
metadata = extract_pdf(example_pdf)
|
||||
assert metadata.sections, 'PDF should produce at least one section'
|
||||
assert all(s.kind == 'pdf_page' for s in metadata.sections)
|
||||
# Pages are 1-indexed; the test PDF has 3 pages.
|
||||
assert [s.ordinal for s in metadata.sections] == [1, 2, 3]
|
||||
assert all(s.label == f'Page {s.ordinal}' for s in metadata.sections)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_docx(test_session, test_directory, example_docx):
|
||||
"""DOCX extractor extracts text content."""
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ import pytest
|
|||
|
||||
from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \
|
||||
get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \
|
||||
search_authors_by_name, search_subjects_by_name
|
||||
search_authors_by_name, search_subjects_by_name, _search_docs
|
||||
from modules.docs.models import Doc, DocSection
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
def test_is_valid_author():
|
||||
|
|
@ -248,3 +250,83 @@ async def test_search_subjects_by_name(test_session):
|
|||
# No match.
|
||||
results = await search_subjects_by_name(test_session, 'nothing')
|
||||
assert len(results) == 0
|
||||
|
||||
|
||||
def _make_doc_with_sections(test_session, test_directory, filename, mimetype, sections):
|
||||
"""Build a FileGroup + Doc + DocSection rows for search tests.
|
||||
|
||||
`sections` is a list of (ordinal, label, content) tuples. The section contents
|
||||
are also concatenated into the FileGroup's d_text so the top-level tsvector
|
||||
search can match the doc in the first place.
|
||||
"""
|
||||
path = test_directory / filename
|
||||
path.write_bytes(b'') # Placeholder; _search_docs doesn't read the file.
|
||||
fg = FileGroup.from_paths(test_session, path)
|
||||
fg.mimetype = mimetype
|
||||
fg.model = 'doc'
|
||||
fg.title = filename
|
||||
fg.a_text = filename
|
||||
fg.d_text = ' '.join(c for _, _, c in sections)
|
||||
test_session.flush()
|
||||
doc = Doc(file_group_id=fg.id, size=1)
|
||||
test_session.add(doc)
|
||||
test_session.flush()
|
||||
kind = 'pdf_page' if mimetype == 'application/pdf' else 'epub_spine'
|
||||
for ordinal, label, content in sections:
|
||||
test_session.add(DocSection(doc_id=doc.id, kind=kind, ordinal=ordinal,
|
||||
label=label, content=content))
|
||||
test_session.commit()
|
||||
return fg, doc
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_docs_attaches_section_hint_epub(test_session, test_directory):
|
||||
"""_search_docs returns a section_hint pointing at the matching spine item."""
|
||||
fg, _ = _make_doc_with_sections(test_session, test_directory,
|
||||
'alpha.epub', 'application/epub+zip', [
|
||||
(0, 'Introduction', 'preface about gardening'),
|
||||
(1, 'Chapter 1', 'a tale of mullen and his dog'),
|
||||
(2, 'Chapter 2', 'unrelated content about pottery'),
|
||||
])
|
||||
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/epub')
|
||||
assert total == 1
|
||||
assert results[0]['id'] == fg.id
|
||||
hint = results[0]['section_hint']
|
||||
assert hint['kind'] == 'epub_spine'
|
||||
assert hint['ordinal'] == 1
|
||||
assert hint['label'] == 'Chapter 1'
|
||||
assert 'mullen' in hint['snippet'].lower()
|
||||
# The snippet uses our sentinel markers rather than raw HTML.
|
||||
assert '[[WROLPI_HL]]' in hint['snippet']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_docs_attaches_section_hint_pdf(test_session, test_directory):
|
||||
"""_search_docs returns a section_hint pointing at the matching PDF page."""
|
||||
fg, _ = _make_doc_with_sections(test_session, test_directory,
|
||||
'beta.pdf', 'application/pdf', [
|
||||
(1, 'Page 1', 'cover page text'),
|
||||
(2, 'Page 2', 'nothing to see here'),
|
||||
(3, 'Page 3', 'here is where mullen shows up'),
|
||||
])
|
||||
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/pdf')
|
||||
assert total == 1
|
||||
assert results[0]['id'] == fg.id
|
||||
hint = results[0]['section_hint']
|
||||
assert hint['kind'] == 'pdf_page'
|
||||
assert hint['ordinal'] == 3
|
||||
assert hint['label'] == 'Page 3'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_docs_no_hint_when_no_query(test_session, test_directory):
|
||||
"""Without a search query, results have no section_hint."""
|
||||
fg, _ = _make_doc_with_sections(test_session, test_directory,
|
||||
'gamma.pdf', 'application/pdf', [
|
||||
(1, 'Page 1', 'whatever'),
|
||||
])
|
||||
results, _ = _search_docs(mimetype='application/pdf')
|
||||
assert results and results[0]['id'] == fg.id
|
||||
assert 'section_hint' not in results[0]
|
||||
|
|
|
|||
|
|
@ -1052,6 +1052,19 @@ def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List
|
|||
results[-1]['c_headline'] = extra['c_headline']
|
||||
results[-1]['d_headline'] = extra['d_headline']
|
||||
|
||||
# When searching by text, attach a `section_hint` to any doc-modeled result so
|
||||
# the UI can deep-link into the matching EPUB chapter or PDF page.
|
||||
search_str = params.get('search_str') if isinstance(params, dict) else None
|
||||
if search_str:
|
||||
from modules.docs.lib import _fetch_section_hints
|
||||
doc_ids = [r['id'] for r in results if r.get('model') == 'doc']
|
||||
if doc_ids:
|
||||
hints = _fetch_section_hints(session, doc_ids, search_str)
|
||||
for r in results:
|
||||
hint = hints.get(r['id'])
|
||||
if hint:
|
||||
r['section_hint'] = hint
|
||||
|
||||
return results, total
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -228,6 +228,34 @@ async def test_files_search(test_session, async_client, make_files_structure, as
|
|||
await assert_files_search('nothing', [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_search_attaches_doc_section_hint(async_client, test_session, test_directory,
|
||||
example_epub, refresh_files):
|
||||
"""The global /api/files/search endpoint attaches `section_hint` to doc-modeled
|
||||
results so the UI can deep-link into the matching chapter. Regression test for a
|
||||
bug where only the docs-specific endpoint (/api/docs/search) attached hints."""
|
||||
await refresh_files()
|
||||
|
||||
body = dict(search_str='chapter')
|
||||
request, response = await async_client.post('/api/files/search', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
epub_results = [r for r in response.json['file_groups']
|
||||
if r.get('model') == 'doc'
|
||||
and (r.get('mimetype') or '').startswith('application/epub')]
|
||||
assert len(epub_results) == 1, f'Expected 1 EPUB result, got: {response.json}'
|
||||
result = epub_results[0]
|
||||
assert 'section_hint' in result, \
|
||||
f'doc result has no section_hint on global search path: {result}'
|
||||
hint = result['section_hint']
|
||||
assert hint['kind'] == 'epub_spine'
|
||||
# The test EPUB's "Chapter one" body is in spine item 1.
|
||||
assert hint['ordinal'] == 1
|
||||
assert hint['label']
|
||||
# Snippet uses our sentinel markers, not raw HTML.
|
||||
assert '[[WROLPI_HL]]' in hint['snippet']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory, refresh_files):
|
||||
one, two = await tag_factory(), await tag_factory()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue