Storing example ebooks for testing.

Special handling of json mimetype for json files.

Moving types up to avoid overwriting mime.types.

Adding custom types to nginx to server ebooks better.

Displaying EBookCard correctly when ebook was not indexed correctly.

Global search can search for all ebooks (mobi/epub).

Testing when mimetype suffix is searched.

Adding eBook to global search filter list.  Opening mimetype search to match mimetypes that may not end with expected suffix.

Adding log warning to explain refresh speed.

Generating ebook covers after refresh.

Testing that another nearby book removes the cover.

Calibre covers can be discovered and associated with an ebook.

Handling if ebooklib is not yet installed.

Adding ebook table and modeler to allow searching the contents of epub files.

Marking all PDFs as un-indexed, so they will be indexed.

Replacing null characters in PDF text.

Testing PDFIndexer against example PDF file.

Installing PyPDF2 to extract text for PDFIndexer.
This commit is contained in:
Roland 2022-12-09 11:22:20 -07:00
parent d77e69f54e
commit 6fcb992540
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
21 changed files with 769 additions and 52 deletions

View file

@ -0,0 +1,66 @@
"""Ebook
Revision ID: 717f36a4da94
Revises: d0bbe45e56c3
Create Date: 2022-12-09 15:32:32.228539
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '717f36a4da94'
down_revision = 'd0bbe45e56c3'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('''
DROP INDEX IF EXISTS ebook_title_idx;
DROP INDEX IF EXISTS ebook_size_idx;
DROP INDEX IF EXISTS ebook_creator_idx;
DROP INDEX IF EXISTS ebook_ebook_path_idx;
DROP INDEX IF EXISTS ebook_cover_path_idx;
DROP TABLE IF EXISTS ebook;
''')
session.execute('''
CREATE TABLE ebook (
id SERIAL PRIMARY KEY,
size INTEGER,
title TEXT,
creator TEXT,
ebook_path TEXT REFERENCES file(path) ON DELETE CASCADE,
cover_path TEXT REFERENCES file(path) ON DELETE CASCADE
)''')
if not DOCKERIZED:
session.execute('ALTER TABLE public.ebook OWNER TO wrolpi')
session.execute('CREATE INDEX ebook_title_idx ON ebook(title)')
session.execute('CREATE INDEX ebook_size_idx ON ebook(size)')
session.execute('CREATE INDEX ebook_creator_idx ON ebook(creator)')
session.execute('CREATE INDEX ebook_ebook_path_idx ON ebook(ebook_path)')
session.execute('CREATE INDEX ebook_cover_path_idx ON ebook(cover_path)')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('''
DROP INDEX IF EXISTS ebook_title_idx;
DROP INDEX IF EXISTS ebook_size_idx;
DROP INDEX IF EXISTS ebook_creator_idx;
DROP INDEX IF EXISTS ebook_ebook_path_idx;
DROP INDEX IF EXISTS ebook_cover_path_idx;
DROP TABLE IF EXISTS ebook;
''')

View file

@ -0,0 +1,30 @@
"""Reindex pdfs
Revision ID: d0bbe45e56c3
Revises: dbf05c8e8dcc
Create Date: 2022-12-09 12:27:18.908533
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = 'd0bbe45e56c3'
down_revision = 'dbf05c8e8dcc'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute("UPDATE file SET indexed=false WHERE mimetype = 'application/pdf'")
def downgrade():
pass

View file

@ -484,10 +484,10 @@ const replaceFileDatetimes = (files) => {
return files;
}
export async function filesSearch(offset, limit, searchStr, mimetype, model) {
export async function filesSearch(offset, limit, searchStr, mimetypes, model) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
if (mimetype) {
body['mimetype'] = mimetype;
if (mimetypes) {
body['mimetypes'] = mimetypes;
}
if (model) {
body['model'] = model;

View file

@ -423,16 +423,24 @@ export function CardGroupCentered(props) {
</div>)
}
export function CardPosterLink({to, poster_url, imageLabel}) {
export function CardPosterLink({to, poster_url, imageLabel, external = false}) {
const {s} = useContext(ThemeContext);
const style = {display: 'flex', justifyContent: 'center', ...s['style']};
return <Link to={to} style={style}>
<Image fluid
src={poster_url}
label={imageLabel}
style={{maxHeight: '163px', width: 'auto'}}
/>
</Link>
const image = <Image fluid
src={poster_url}
label={imageLabel}
style={{maxHeight: '163px', width: 'auto'}}
/>;
if (external === true) {
return <a href={to} target='_blank' style={style}>
{image}
</a>
} else {
// Link using React Router.
return <Link to={to} style={style}>
{image}
</Link>
}
}
export function HelpPopup({icon, size, content, position}) {
@ -567,6 +575,9 @@ export function mimetypeColor(mimetype) {
return 'green'
} else if (mimetype && mimetype.startsWith('application/zip')) {
return 'purple'
} else if (mimetype && (
mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook'))) {
return 'yellow'
}
} catch (e) {
console.error(e);
@ -595,6 +606,8 @@ export function FileIcon({file, disabled = true, size = 'huge', ...props}) {
props['name'] = 'file archive';
} else if (mimetype.startsWith('application/x-iso9660-image')) {
props['name'] = 'dot circle';
} else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook')) {
props['name'] = 'book';
}
}
return <Icon disabled={disabled} size={size} {...props}/>
@ -763,3 +776,19 @@ export const ColorToSemanticHexColor = (color) => {
}
return colorMap[color] || null;
}
export const filterToMimetypes = (filter) => {
if (filter === 'video') {
return ['video'];
} else if (filter === 'archive') {
return ['text/html'];
} else if (filter === 'pdf') {
return ['application/pdf'];
} else if (filter === 'ebook') {
return ['application/epub+zip', 'application/x-mobipocket-ebook'];
} else if (filter === 'image') {
return ['image'];
} else if (filter === 'zip') {
return ['application/zip'];
}
}

View file

@ -6,6 +6,7 @@ import {
CardHeader,
CardMeta,
Confirm,
Container,
Divider,
Dropdown,
Image,
@ -17,6 +18,8 @@ import 'react-keyed-file-browser/dist/react-keyed-file-browser.css';
import {deleteFile, refreshDirectoryFiles, refreshFiles} from "../api";
import {
CardGroupCentered,
CardPosterLink,
epochToDateString,
ExternalCardLink,
FileIcon,
humanFileSize,
@ -25,7 +28,6 @@ import {
PageContainer,
Paginator,
textEllipsis,
epochToDateString,
useTitle
} from "./Common";
import {useBrowseFiles, useQuery, useSearchFiles} from "../hooks/customHooks";
@ -37,6 +39,7 @@ import {ThemeContext} from "../contexts/contexts";
import {Button, Card, CardIcon, Header, Icon, Placeholder, Segment} from "./Theme";
import {SelectableTable} from "./Tables";
import {VideoCard, VideoRowCells} from "./Videos";
import _ from 'lodash';
const icons = {
File: <Icon name='file'/>,
@ -142,6 +145,42 @@ export function Files() {
</>
}
function EbookCard({file}) {
const {s} = useContext(ThemeContext);
const url = `/media/${encodeURIComponent(file.path)}`;
let {ebook, suffix} = file;
const coverSrc = ebook && ebook.cover_path ? `/media/${encodeURIComponent(ebook.cover_path)}` : null;
let cover = <CardIcon><FileIcon file={file}/></CardIcon>;
if (coverSrc) {
cover = <CardPosterLink to={url} poster_url={coverSrc} external={true}/>;
}
suffix = suffix ? _.trimStart(suffix, '.').toUpperCase() : null;
const color = mimetypeColor(file.mimetype);
return <Card color={color}>
{cover}
<CardContent {...s}>
<CardHeader>
<Container textAlign='left'>
<ExternalCardLink to={url}>{ebook ? ebook.title : file.title}</ExternalCardLink>
</Container>
</CardHeader>
<CardDescription>
<Container textAlign='left'>
<b {...s}>{ebook ? ebook.creator : null}</b>
</Container>
</CardDescription>
<CardMeta>
<p {...s}>{ebook ? humanFileSize(ebook.size) : null}</p>
<pre {...s}>{suffix}</pre>
</CardMeta>
</CardContent>
</Card>
}
function ImageCard({file}) {
const {s} = useContext(ThemeContext);
const url = `/media/${encodeURIComponent(file.path)}`;
@ -184,6 +223,10 @@ function FileCard({file}) {
return <ArchiveCard key={file['path']} file={file}/>;
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
return <ImageCard key={file['path']} file={file}/>;
} else if (file.mimetype && (
file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
)) {
return <EbookCard key={file['path']} file={file}/>;
}
const url = `/media/${encodeURIComponent(file.path)}`;
@ -399,26 +442,27 @@ export function FilesSearchView({
showSelect = false,
emptySearch = false,
filterOptions,
mimetype,
model,
onSelect,
setFilters,
}) {
const {searchFiles, limit, setLimit, totalPages, activePage, setPage} =
useSearchFiles(24, emptySearch, mimetype, model);
filterOptions = filterOptions || [
{key: 'video', text: 'Video', value: 'video'},
{key: 'archive', text: 'Archive', value: 'archive'},
{key: 'pdf', text: 'PDF', value: 'pdf'},
{key: 'ebook', text: 'eBook', value: 'ebook'},
{key: 'image', text: 'Image', value: 'image'},
{key: 'zip', text: 'ZIP', value: 'zip'},
];
const {searchFiles, limit, setLimit, totalPages, activePage, setPage, filter} =
useSearchFiles(24, emptySearch, model);
const {searchParams, updateQuery} = useQuery();
const setView = (value) => updateQuery({view: value});
const view = searchParams.get('view');
const setMimetype = (value) => updateQuery({'mimetype': value});
filterOptions = filterOptions || [
{key: 'video', text: 'Video', value: 'video'},
{key: 'archive', text: 'Archive', value: 'text/html'},
{key: 'pdf', text: 'PDF', value: 'application/pdf'},
{key: 'zip', text: 'ZIP', value: 'application/zip'},
{key: 'image', text: 'Image', value: 'image'},
];
const setFilter = (value) => updateQuery({'filter': value});
return <FilesView
files={searchFiles}
@ -434,7 +478,7 @@ export function FilesSearchView({
onSelect={onSelect}
setPage={setPage}
filterOptions={filterOptions}
setFilters={setFilters || setMimetype}
setFilters={setFilters || setFilter}
/>
}

View file

@ -25,7 +25,7 @@ import {
} from "../api";
import {createSearchParams, useSearchParams} from "react-router-dom";
import {toast} from "react-semantic-toasts";
import {enumerate, humanFileSize, secondsToFullDuration} from "../components/Common";
import {enumerate, filterToMimetypes, humanFileSize, secondsToFullDuration} from "../components/Common";
import {StatusContext} from "../contexts/contexts";
const calculatePage = (offset, limit) => {
@ -386,12 +386,12 @@ export const useChannels = () => {
return {channels, fetchChannels}
}
export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype, model) => {
export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => {
const {searchParams, updateQuery} = useQuery();
const limit = searchParams.get('l') || defaultLimit;
const offset = searchParams.get('o') || 0;
const searchStr = searchParams.get('q');
const mimetype_ = searchParams.get('mimetype');
const filter = searchParams.get('filter');
const model_ = searchParams.get('model');
const [searchFiles, setSearchFiles] = useState(null);
@ -402,11 +402,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype,
if (!emptySearch && !searchStr) {
return;
}
const mimetypes = filterToMimetypes(filter);
setSearchFiles(null);
setTotalPages(0);
try {
let [files, total] = await filesSearch(
offset, limit, searchStr, mimetype || mimetype_, model || model_);
offset, limit, searchStr, mimetypes, model || model_);
setSearchFiles(files);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -422,7 +423,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype,
useEffect(() => {
localSearchFiles();
}, [searchStr, limit, offset, activePage, mimetype, mimetype_, model, model_]);
}, [searchStr, limit, offset, activePage, filter, model, model_]);
const setPage = (i) => {
i = parseInt(i);
@ -440,7 +441,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype,
updateQuery({l: value, o: 0});
}
return {searchFiles, totalPages, limit, searchStr, mimetype, setSearchStr, activePage, setPage, setLimit};
return {searchFiles, totalPages, limit, searchStr, filter, setSearchStr, activePage, setPage, setLimit};
}
export const useBrowseFiles = () => {

View file

@ -10,7 +10,13 @@ http {
'' close;
}
charset utf-8;
include mime.types;
types {
# Custom types for ebooks.
application/x-mobipocket-ebook mobi;
application/epub+zip epub;
}
error_page 502 /error/50x.html;
error_page 503 /error/50x.html;
@ -54,6 +60,7 @@ http {
}
location /media {
# The media files mounted in the docker volume
sendfile on;
sendfile_max_chunk 1m;

View file

@ -2,13 +2,20 @@ events {
}
http {
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
charset utf-8;
include mime.types;
types {
# Custom types for ebooks.
application/x-mobipocket-ebook mobi;
application/epub+zip epub;
}
server {
location /favicon.ico {
@ -43,6 +50,7 @@ http {
}
location /media {
# The media files mounted in the docker volume
sendfile on;
sendfile_max_chunk 1m;

View file

@ -1,11 +1,13 @@
Pillow==9.3.0
Pint==0.18
PyPDF2[crypto]==2.11.2
PyYAML==5.3.1 # this is the only version available on RPi
SQLAlchemy==1.3.22
aiohttp==3.8.1
alembic==1.7.5
beautifulsoup4==4.10.0
cachetools==5.0.0
ebooklib==0.18
feedparser==6.0.9
mock==4.0.3
psutil==5.9.2

BIN
test/example.epub Normal file

Binary file not shown.

BIN
test/example.mobi Normal file

Binary file not shown.

BIN
test/example.pdf Normal file

Binary file not shown.

View file

@ -231,6 +231,27 @@ def bad_vtt_file(test_directory) -> pathlib.Path:
yield destination
@pytest.fixture
def example_pdf(test_directory) -> pathlib.Path:
destination = test_directory / 'example.pdf'
shutil.copy(PROJECT_DIR / 'test/example.pdf', destination)
yield destination
@pytest.fixture
def example_epub(test_directory) -> pathlib.Path:
destination = test_directory / 'example.epub'
shutil.copy(PROJECT_DIR / 'test/example.epub', destination)
yield destination
@pytest.fixture
def example_mobi(test_directory) -> pathlib.Path:
destination = test_directory / 'example.mobi'
shutil.copy(PROJECT_DIR / 'test/example.mobi', destination)
yield destination
@pytest.fixture
def make_files_structure(test_directory) -> Callable[[Union[List, Dict]], List[pathlib.Path]]:
def create_files(paths: List) -> List[pathlib.Path]:
@ -277,7 +298,7 @@ def wrol_mode_fixture(test_config, test_download_manager):
@pytest.fixture
def mock_create_subprocess_shell():
def mocker(communicate_return=None, return_code=None, communicate_side_effect = None):
def mocker(communicate_return=None, return_code=None, communicate_side_effect=None):
async def communicate(*a, **kw):
if communicate_side_effect:
return communicate_side_effect()

View file

@ -1 +1,2 @@
from . import ebooks # noqa
from .api import bp

View file

@ -115,7 +115,7 @@ async def refresh_files_list(_: Request, body: schema.FilesRefreshListRequest):
)
@validate(schema.FilesSearchRequest)
async def post_search_files(_: Request, body: schema.FilesSearchRequest):
files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetype, body.model)
files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model)
return json_response(dict(files=files, totals=dict(files=total)))

303
wrolpi/files/ebooks.py Normal file
View file

@ -0,0 +1,303 @@
import dataclasses
import logging
import pathlib
from typing import Dict, List, Generator, Optional
import bs4
from sqlalchemy import Column, Integer, ForeignKey, String
from sqlalchemy.orm import Session, relationship
from wrolpi.common import register_modeler, ModelHelper, Base, register_after_refresh
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
from wrolpi.files.models import File
from wrolpi.media_path import MediaPathType
try:
import ebooklib
from ebooklib import epub
except ImportError:
ebooklib = None
epub = None
logger = logging.getLogger(__name__)
__all__ = [
'EBook',
'EBookData',
'extract_ebook_cover',
'extract_ebook_data',
'ebook_modeler',
'MOBI_MIMETYPE',
'EPUB_MIMETYPE'
]
EBOOK_SUFFIXES = ('.epub', '.mobi')
EPUB_MIMETYPE = 'application/epub'
MOBI_MIMETYPE = 'application/x-mobipocket-ebook'
EBOOK_MIMETYPES = (
EPUB_MIMETYPE,
MOBI_MIMETYPE,
)
@dataclasses.dataclass
class EBookData:
cover: bytes = None
creator: str = None
text: str = None
title: str = None
def __bool__(self):
return bool(self.cover) or bool(self.creator) or bool(self.text) or bool(self.title)
def extract_text(html: str) -> str:
soup = bs4.BeautifulSoup(html, features='html.parser')
text = soup.get_text()
return text
def mimetype_is_ebook(mimetype: str) -> bool:
return any(mimetype.startswith(i) for i in EBOOK_MIMETYPES)
def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]:
"""Extract data from within an eBook file."""
if ebooklib is None or epub is None:
raise ValueError('ebooklib is not installed')
if not mimetype.startswith(EPUB_MIMETYPE):
return None
data = EBookData()
book = epub.read_epub(path, options=dict(ignore_ncx=True))
for key, value in book.metadata.items():
if 'title' in value:
data.title = data.title or value['title'][0][0]
if 'creator' in value:
data.creator = data.creator or value['creator'][0][0]
data.text = ''
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
data.text += extract_text(doc.content)
return data
def extract_ebook_cover(path: pathlib.Path) -> Optional[bytes]:
"""Attempt to find the Cover within an ePub file. If no cover is found, return the first image."""
if ebooklib is None or epub is None:
raise ValueError('ebooklib is not installed')
book = epub.read_epub(path, options=dict(ignore_ncx=True))
for doc in book.get_items_of_type(ebooklib.ITEM_COVER):
logger.debug(f'Found cover for {path}')
return doc.content
# Use the first image as the cover.
for doc in book.get_items_of_type(ebooklib.ITEM_IMAGE):
logger.debug(f'Using first image as cover for {path}')
return doc.content
def discover_calibre_cover(ebook_path: pathlib.Path):
"""Calibre puts a cover file in a directory which contains only one ebook (but multiple formats of the ebook),
this function will return the path of the cover if the ebook_path is in such a situation."""
if not ebook_path.is_file():
raise ValueError(f'Invalid ebook path {ebook_path}')
paths = list(ebook_path.parent.iterdir())
ebook_path_stem, _ = split_path_stem_and_suffix(ebook_path)
cover = None
for path in paths:
stem, suffix = split_path_stem_and_suffix(path)
if path.is_dir():
# We don't care about sub directories.
continue
if stem == ebook_path_stem:
# This is one of the formats of the ebook.
continue
if path.name == 'metadata.opf':
# Metadata, this is expected in a Calibre book directory.
continue
if stem == 'cover' and get_mimetype(path).startswith('image/'):
# Discovered the cover.
cover = path
continue
# Some file that does not match the ebook stem, metadata, or cover. This must not be a Calibre book directory.
return None
return cover
class EBook(ModelHelper, Base):
__tablename__ = 'ebook'
id = Column(Integer, primary_key=True)
size: int = Column(Integer)
title: str = Column(String)
creator: str = Column(String)
ebook_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
ebook_file: File = relationship('File', primaryjoin='EBook.ebook_path==File.path')
cover_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
cover_file: File = relationship('File', primaryjoin='EBook.cover_path==File.path')
def __repr__(self):
cover = 'yes' if self.cover_file else 'no'
return f'<EBook id={self.id} path={repr(self.ebook_path)} cover={cover}>'
def my_paths(self) -> Generator[pathlib.Path, None, None]:
if self.ebook_path:
yield self.ebook_path
if self.cover_path:
yield self.cover_path
def my_files(self) -> Generator[File, None, None]:
if self.ebook_file:
yield self.ebook_file
if self.cover_file:
yield self.cover_file
def __json__(self):
d = dict(
cover_path=self.cover_path,
creator=self.creator,
ebook_path=self.ebook_path,
id=self.id,
size=self.size,
title=self.title,
)
return d
def generate_cover(self) -> Optional[pathlib.Path]:
"""Discover this ebook's cover, if found, store it next to the ebook file."""
path = self.ebook_file.path if self.ebook_file else self.ebook_path
if not path:
raise ValueError('Cannot generate cover for ebook without ebook file!')
if calibre_cover := discover_calibre_cover(path):
return calibre_cover
cover_bytes = extract_ebook_cover(path)
if cover_bytes:
cover_path = path.with_suffix('.jpeg')
cover_path.write_bytes(cover_bytes)
return cover_path
logger.warning(f'Unable to generate cover for {self.ebook_path}')
@staticmethod
def find_by_path(path, session) -> Base:
return session.query(EBook).filter(EBook.ebook_path == path).one_or_none()
def model_ebook(session: Session, ebook_file: File, files: List[File]) -> EBook:
"""
Creates an EBook model based off a File. Searches for it's cover in the provided `files`.
"""
# Multiple formats may share this cover.
cover_file = next((i for i in files if i.mimetype.split('/')[0] == 'image'), None)
ebook = session.query(EBook).filter_by(ebook_file=ebook_file).one_or_none()
if not ebook:
ebook = EBook(ebook_file=ebook_file)
session.add(ebook)
size = ebook_file.path.stat().st_size
ebook.ebook_file.do_index()
index = size != ebook.size or cover_file != ebook.cover_file or not ebook.title
if not ebook.ebook_file.indexed or index:
try:
# Only read the contents of the file if it has changed.
data = extract_ebook_data(ebook_file.path, ebook_file.mimetype)
if data:
# Title is a_text.
ebook.ebook_file.a_text = ebook.title = data.title
# Creator is b_text.
ebook.ebook_file.b_text = ebook.creator = data.creator
# All text is d_text.
ebook.ebook_file.d_text = data.text
except Exception as e:
logger.error(f'Failed to extract book data', exc_info=e)
if not ebook.title:
# Book was not indexed above (probably a MOBI), use the file data.
name, _ = split_path_stem_and_suffix(ebook.ebook_file.path)
ebook.title = ebook.ebook_file.a_text = name
if cover_file:
cover_file.associated = True
ebook.cover_file = cover_file
ebook.size = size
ebook_file.model = EBook.__tablename__
return ebook
@register_modeler
def ebook_modeler(groups: Dict[str, List[File]], session: Session):
"""Searches for ebook files and models them into the ebook table.
May generate cover files for EPUBs."""
local_groups = groups.copy()
for stem, group in local_groups.items():
found_ebook = False
for file in group:
if not mimetype_is_ebook(file.mimetype):
continue
session.flush(group)
model_ebook(session, file, group)
found_ebook = True
if found_ebook:
# Claim this group for this ebook.
del groups[stem]
@register_after_refresh
def discover_ebook_covers():
"""Discovers/extracts ebook covers for any Ebook that is missing one."""
with get_db_curs() as curs:
stmt = f'''
SELECT e.id
FROM ebook e
LEFT JOIN file f on e.ebook_path = f.path
WHERE
e.cover_path IS NULL
AND f.mimetype LIKE '{EPUB_MIMETYPE}%' '''
curs.execute(stmt)
missing_covers = [i[0] for i in curs.fetchall()]
if not missing_covers:
logger.info('No ebook covers to generate')
return
logger.warning(f'Discovering/generating {len(missing_covers)} ebook covers')
for id_ in missing_covers:
with get_db_session(commit=True) as session:
ebook: EBook = session.query(EBook).filter(EBook.id == id_).one()
cover_path = ebook.generate_cover()
if cover_path:
# Cover was generated or discovered. Associate it with the ebook.
cover_file = session.query(File).filter(File.path == cover_path).one_or_none()
if not cover_file:
cover_file = File(path=cover_path)
session.add(cover_file)
cover_file.do_index()
cover_file.associated = True
ebook.cover_file = cover_file

View file

@ -5,6 +5,11 @@ from collections import defaultdict
from typing import List, Type, Tuple
from zipfile import ZipFile
try:
from PyPDF2 import PdfReader
except ImportError:
PdfReader = None
from wrolpi.common import logger
logger = logger.getChild(__name__)
@ -45,10 +50,10 @@ def find_indexer(file) -> Type[Indexer]:
if not file.mimetype:
return DefaultIndexer
if file.mimetype in indexer_map:
# Use the specific indexer.
indexer = indexer_map[file.mimetype]
else:
# Get the indexer that matches the mimetype of this file.
indexer = next((v for k, v in indexer_map.items() if file.mimetype.startswith(k)), None)
if not indexer:
# Use the broad indexer.
indexer = indexer_map[file.mimetype.split('/')[0]]
@ -126,3 +131,36 @@ class TextIndexer(Indexer, ABC):
# Some text files aren't for humans.
return DefaultIndexer
return cls
@register_indexer('application/pdf')
class PDFIndexer(Indexer, ABC):
"""Uses PyPDF2 to extract text from a PDF."""
@classmethod
def create_index(cls, file):
path = file.path.path if hasattr(file.path, 'path') else file.path
a = cls.get_title(file)
try:
# PDFs are complex, don't fail to create title index if text extraction fails.
words = cls.get_words(path)
except Exception as e:
logger.error(f'Failed to index {path}', exc_info=e)
words = ''
return a, None, None, words
@classmethod
def get_words(cls, path: pathlib.Path) -> str:
if PdfReader is None:
return ''
reader = PdfReader(path)
words = []
for page in reader.pages:
# Get all text facing up.
text = page.extract_text(0)
words.append(text)
words = '\n'.join(words)
# Postgres does not allow null characters.
words = words.replace('\x00', '\uFFFD')
return words

View file

@ -28,11 +28,10 @@ try:
import magic
mime = magic.Magic(mime=True)
no_magic = False
except ImportError:
# Magic is not installed
no_magic = True
magic = None
mime = None
logger = logger.getChild(__name__)
@ -95,9 +94,20 @@ FILE_NAME_REGEX = re.compile(r'[_ .]')
FILE_BIN = which('file', '/usr/bin/file')
def _mimetype_suffix_map(path: Path, mimetype: str):
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
from wrolpi.files.ebooks import MOBI_MIMETYPE
suffix = path.suffix.lower()
if mimetype == 'application/octet-stream' and suffix.endswith('.mobi'):
return MOBI_MIMETYPE
if mimetype == 'text/plain' and suffix.endswith('.json'):
return 'application/json'
return mimetype
def get_mimetype(path: Path) -> str:
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
if no_magic:
if magic is None:
# This method is slow, prefer the speedier `magic` module.
cmd = (FILE_BIN, '--mime-type', str(path.absolute()))
output = subprocess.check_output(cmd)
@ -105,7 +115,9 @@ def get_mimetype(path: Path) -> str:
mimetype = output.split(' ')[-1].strip()
return mimetype
else:
return mime.from_file(path)
mimetype = mime.from_file(path)
mimetype = _mimetype_suffix_map(path, mimetype)
return mimetype
# Special suffixes within WROLPi.
@ -328,17 +340,18 @@ async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str
refresh_logger.info(f'Done refreshing files in {directory}')
def search_files(search_str: str, limit: int, offset: int, mimetype: str = None, model: str = None) -> \
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None) -> \
Tuple[List[dict], int]:
"""
Search the Files table. Order the returned Files by their rank if `search_str` is provided. Return all files if
"""Search the Files table.
Order the returned Files by their rank if `search_str` is provided. Return all files if
`search_str` is empty.
Parameters:
search_str: Search the ts_vector of the file. Returns all files if this is empty.
limit: Return only this many files.
offset: Offset the query.
mimetype: Only return files that match this mimetype.
mimetypes: Only return files that match these mimetypes.
model: Only return files that match this model.
"""
params = dict(offset=offset, limit=limit)
@ -352,12 +365,17 @@ def search_files(search_str: str, limit: int, offset: int, mimetype: str = None,
selects.append('ts_rank(textsearch, websearch_to_tsquery(%(search_str)s))')
order_by = '2 DESC'
if mimetype and len(mimetype.split('/')) == 1:
params['mimetype'] = f'{mimetype}/%'
wheres.append('mimetype LIKE %(mimetype)s')
elif mimetype:
wheres.append('mimetype = %(mimetype)s')
params['mimetype'] = mimetype
if mimetypes:
mimetype_wheres = []
for idx, mimetype in enumerate(mimetypes):
key = f'mimetype{idx}'
if len(mimetype.split('/')) == 1:
params[key] = f'{mimetype}/%'
else:
params[key] = f'{mimetype}%'
mimetype_wheres.append(f'mimetype LIKE %({key})s')
mimetype_wheres = ' OR '.join(mimetype_wheres)
wheres.append(f'({mimetype_wheres})')
if model:
params['model'] = model

View file

@ -17,7 +17,7 @@ class FilesSearchRequest:
search_str: Optional[str] = None
limit: Optional[int] = 20
offset: Optional[int] = 0
mimetype: Optional[str] = None
mimetypes: List[str] = field(default_factory=list)
model: Optional[str] = None

View file

@ -0,0 +1,127 @@
import json
import shutil
from http import HTTPStatus
import pytest
from wrolpi.files.ebooks import EBook, MOBI_MIMETYPE, EPUB_MIMETYPE
from wrolpi.files.lib import refresh_files
from wrolpi.test.common import assert_dict_contains
@pytest.mark.asyncio
async def test_index(test_session, test_directory, example_epub, example_mobi):
"""Ebooks can be indexed.
Covers can be discovered."""
await refresh_files()
ebook_epub, ebook_mobi = test_session.query(EBook).order_by(EBook.ebook_path)
ebook_epub: EBook
ebook_mobi: EBook
assert ebook_epub.title == 'WROLPi Test Book'
assert ebook_epub.ebook_file.mimetype.startswith(EPUB_MIMETYPE)
assert ebook_epub.creator == 'roland'
assert ebook_epub.size == ebook_epub.ebook_file.size
assert ebook_epub.cover_path and ebook_epub.cover_path.is_file()
# The cover is the WROLPi logo.
assert ebook_epub.cover_file.size == 297099
assert ebook_epub.ebook_file.a_text, 'Book title was not indexed'
assert ebook_epub.ebook_file.b_text, 'Book creator was not indexed'
assert ebook_epub.ebook_file.d_text, 'Book text was not indexed'
# Mobi is not fully supported, title is the file name. No creator or cover.
assert ebook_mobi.title == 'example'
assert ebook_mobi.ebook_file.mimetype == MOBI_MIMETYPE
assert not ebook_mobi.creator
assert not ebook_mobi.cover_path
# Ebooks can be deleted during refresh.
example_mobi.unlink()
example_epub.unlink()
await refresh_files()
assert test_session.query(EBook).count() == 0
def test_search(test_session, test_client, example_epub):
"""Ebooks are handled in File search results."""
request, response = test_client.post('/api/files/refresh')
assert response.status == HTTPStatus.NO_CONTENT
assert test_session.query(EBook).count() == 1
ebook: EBook = test_session.query(EBook).one()
assert ebook.title == 'WROLPi Test Book'
assert ebook.creator == 'roland'
assert ebook.ebook_file.a_text == 'WROLPi Test Book', 'Book title was not updated'
assert ebook.ebook_file.d_text, 'Book was not indexed'
content = dict(mimetypes=['application/epub', 'application/x-mobipocket-ebook'])
request, response = test_client.post('/api/files/search', content=json.dumps(content))
assert response.status == HTTPStatus.OK
assert response.json
result = response.json['files'][0]
assert result['path'] == 'example.epub'
assert result['mimetype'] == 'application/epub+zip'
assert_dict_contains(
result['ebook'],
{'cover_path': 'example.jpeg', 'ebook_path': 'example.epub'},
)
content = dict(mimetypes=['application/epub+zip', 'application/x-mobipocket-ebook'])
request, response = test_client.post('/api/files/search', content=json.dumps(content))
result = response.json['files'][0]
assert result['path'] == 'example.epub'
assert result['mimetype'] == 'application/epub+zip'
content = dict(mimetypes=[])
request, response = test_client.post('/api/files/search', content=json.dumps(content))
assert response.status == HTTPStatus.OK
assert response.json
@pytest.mark.asyncio
async def test_discover_calibre_cover(test_session, test_directory, example_epub, image_file):
"""Calibre puts a cover near an ebook file, test if it can be found."""
# Create two files in the
metadata = test_directory / 'metadata.opf'
metadata.touch()
# Create an alternate format next to the epub. This will be ignored because it is not yet supported.
mobi_path = example_epub.with_suffix('.mobi')
mobi_path.touch()
await refresh_files()
assert test_session.query(EBook).count() == 1
ebook: EBook = test_session.query(EBook).one()
assert ebook.title == 'WROLPi Test Book'
# The cover in the ebook was extracted.
assert ebook.cover_file.size == 297099
# Delete the extracted cover, use the cover image from Calibre.
ebook.cover_path.unlink()
test_session.delete(ebook.cover_file)
cover_image = test_directory / 'cover.jpg'
shutil.move(image_file, cover_image)
# Calibre cover image was discovered, no cover was generated.
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
assert ebook.title == 'WROLPi Test Book'
assert ebook.cover_file.size == 641
# Create a file nearby, its no longer a Calibre directory.
mobi_path = mobi_path.rename(test_directory / 'other book.mobi')
await refresh_files()
assert ebook.title == 'WROLPi Test Book'
assert ebook.cover_file.size == 297099
mobi_path.unlink()
# Delete the discovered cover, cover should be generated.
cover_image.unlink()
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
assert ebook.title == 'WROLPi Test Book'
assert ebook.cover_file.size == 297099

View file

@ -469,3 +469,25 @@ def test_matching_directories(make_files_structure, test_directory):
# foo is an exact match, return subdirectories
matches = lib.get_matching_directories(test_directory / 'foo')
assert matches == [str(test_directory / 'foo/qux')]
def test_pdf_indexer(example_pdf):
"""PDFs can be indexed by PDFIndexer."""
file = File(path=example_pdf)
a_text, b_text, c_text, d_text = indexers.PDFIndexer.create_index(file)
# The file name.
assert a_text == ('example', '.pdf')
assert b_text is None
assert c_text is None
# Both pages of the example PDF are extracted.
assert d_text == 'Page one\nPage two'
def test_get_mimetype(example_epub, example_mobi, example_pdf, image_file, video_file):
assert lib.get_mimetype(example_epub) == 'application/epub+zip'
assert lib.get_mimetype(example_mobi) == 'application/x-mobipocket-ebook'
assert lib.get_mimetype(example_pdf) == 'application/pdf'
assert lib.get_mimetype(image_file) == 'image/jpeg'
assert lib.get_mimetype(video_file) == 'video/mp4'