diff --git a/alembic/versions/717f36a4da94_ebook.py b/alembic/versions/717f36a4da94_ebook.py new file mode 100644 index 00000000..f45e396a --- /dev/null +++ b/alembic/versions/717f36a4da94_ebook.py @@ -0,0 +1,66 @@ +"""Ebook + +Revision ID: 717f36a4da94 +Revises: d0bbe45e56c3 +Create Date: 2022-12-09 15:32:32.228539 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '717f36a4da94' +down_revision = 'd0bbe45e56c3' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute(''' + DROP INDEX IF EXISTS ebook_title_idx; + DROP INDEX IF EXISTS ebook_size_idx; + DROP INDEX IF EXISTS ebook_creator_idx; + DROP INDEX IF EXISTS ebook_ebook_path_idx; + DROP INDEX IF EXISTS ebook_cover_path_idx; + DROP TABLE IF EXISTS ebook; + ''') + + session.execute(''' + CREATE TABLE ebook ( + id SERIAL PRIMARY KEY, + size INTEGER, + title TEXT, + creator TEXT, + ebook_path TEXT REFERENCES file(path) ON DELETE CASCADE, + cover_path TEXT REFERENCES file(path) ON DELETE CASCADE + )''') + + if not DOCKERIZED: + session.execute('ALTER TABLE public.ebook OWNER TO wrolpi') + + session.execute('CREATE INDEX ebook_title_idx ON ebook(title)') + session.execute('CREATE INDEX ebook_size_idx ON ebook(size)') + session.execute('CREATE INDEX ebook_creator_idx ON ebook(creator)') + session.execute('CREATE INDEX ebook_ebook_path_idx ON ebook(ebook_path)') + session.execute('CREATE INDEX ebook_cover_path_idx ON ebook(cover_path)') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute(''' + DROP INDEX IF EXISTS ebook_title_idx; + DROP INDEX IF EXISTS ebook_size_idx; + DROP INDEX IF EXISTS ebook_creator_idx; + DROP INDEX IF EXISTS ebook_ebook_path_idx; + DROP INDEX IF EXISTS ebook_cover_path_idx; + DROP TABLE IF EXISTS ebook; + ''') diff --git a/alembic/versions/d0bbe45e56c3_reindex_pdfs.py b/alembic/versions/d0bbe45e56c3_reindex_pdfs.py new file mode 100644 index 00000000..3c584296 --- /dev/null +++ b/alembic/versions/d0bbe45e56c3_reindex_pdfs.py @@ -0,0 +1,30 @@ +"""Reindex pdfs + +Revision ID: d0bbe45e56c3 +Revises: dbf05c8e8dcc +Create Date: 2022-12-09 12:27:18.908533 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'd0bbe45e56c3' +down_revision = 'dbf05c8e8dcc' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute("UPDATE file SET indexed=false WHERE mimetype = 'application/pdf'") + + +def downgrade(): + pass diff --git a/app/src/api.js b/app/src/api.js index 0f1f8a2e..7f32a504 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -484,10 +484,10 @@ const replaceFileDatetimes = (files) => { return files; } -export async function filesSearch(offset, limit, searchStr, mimetype, model) { +export async function filesSearch(offset, limit, searchStr, mimetypes, model) { const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)}; - if (mimetype) { - body['mimetype'] = mimetype; + if (mimetypes) { + body['mimetypes'] = mimetypes; } if (model) { body['model'] = model; diff --git a/app/src/components/Common.js b/app/src/components/Common.js index bc70f895..42126b1c 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -423,16 +423,24 @@ export function CardGroupCentered(props) { ) } -export function CardPosterLink({to, poster_url, imageLabel}) { +export function CardPosterLink({to, poster_url, imageLabel, external = false}) { const {s} = useContext(ThemeContext); const style = {display: 'flex', justifyContent: 'center', ...s['style']}; - return - - + const image = ; + if (external === true) { + return + {image} + + } else { + // Link using React Router. + return + {image} + + } } export function HelpPopup({icon, size, content, position}) { @@ -567,6 +575,9 @@ export function mimetypeColor(mimetype) { return 'green' } else if (mimetype && mimetype.startsWith('application/zip')) { return 'purple' + } else if (mimetype && ( + mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook'))) { + return 'yellow' } } catch (e) { console.error(e); @@ -595,6 +606,8 @@ export function FileIcon({file, disabled = true, size = 'huge', ...props}) { props['name'] = 'file archive'; } else if (mimetype.startsWith('application/x-iso9660-image')) { props['name'] = 'dot circle'; + } else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook')) { + props['name'] = 'book'; } } return @@ -763,3 +776,19 @@ export const ColorToSemanticHexColor = (color) => { } return colorMap[color] || null; } + +export const filterToMimetypes = (filter) => { + if (filter === 'video') { + return ['video']; + } else if (filter === 'archive') { + return ['text/html']; + } else if (filter === 'pdf') { + return ['application/pdf']; + } else if (filter === 'ebook') { + return ['application/epub+zip', 'application/x-mobipocket-ebook']; + } else if (filter === 'image') { + return ['image']; + } else if (filter === 'zip') { + return ['application/zip']; + } +} diff --git a/app/src/components/Files.js b/app/src/components/Files.js index ee75cb22..dfa63610 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -6,6 +6,7 @@ import { CardHeader, CardMeta, Confirm, + Container, Divider, Dropdown, Image, @@ -17,6 +18,8 @@ import 'react-keyed-file-browser/dist/react-keyed-file-browser.css'; import {deleteFile, refreshDirectoryFiles, refreshFiles} from "../api"; import { CardGroupCentered, + CardPosterLink, + epochToDateString, ExternalCardLink, FileIcon, humanFileSize, @@ -25,7 +28,6 @@ import { PageContainer, Paginator, textEllipsis, - epochToDateString, useTitle } from "./Common"; import {useBrowseFiles, useQuery, useSearchFiles} from "../hooks/customHooks"; @@ -37,6 +39,7 @@ import {ThemeContext} from "../contexts/contexts"; import {Button, Card, CardIcon, Header, Icon, Placeholder, Segment} from "./Theme"; import {SelectableTable} from "./Tables"; import {VideoCard, VideoRowCells} from "./Videos"; +import _ from 'lodash'; const icons = { File: , @@ -142,6 +145,42 @@ export function Files() { > } +function EbookCard({file}) { + const {s} = useContext(ThemeContext); + const url = `/media/${encodeURIComponent(file.path)}`; + let {ebook, suffix} = file; + const coverSrc = ebook && ebook.cover_path ? `/media/${encodeURIComponent(ebook.cover_path)}` : null; + + let cover = ; + if (coverSrc) { + cover = ; + } + + suffix = suffix ? _.trimStart(suffix, '.').toUpperCase() : null; + + const color = mimetypeColor(file.mimetype); + return + {cover} + + + + {ebook ? ebook.title : file.title} + + + + + {ebook ? ebook.creator : null} + + + + {ebook ? humanFileSize(ebook.size) : null} + {suffix} + + + + +} + function ImageCard({file}) { const {s} = useContext(ThemeContext); const url = `/media/${encodeURIComponent(file.path)}`; @@ -184,6 +223,10 @@ function FileCard({file}) { return ; } else if (file.mimetype && file.mimetype.startsWith('image/')) { return ; + } else if (file.mimetype && ( + file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook') + )) { + return ; } const url = `/media/${encodeURIComponent(file.path)}`; @@ -399,26 +442,27 @@ export function FilesSearchView({ showSelect = false, emptySearch = false, filterOptions, - mimetype, model, onSelect, setFilters, }) { - const {searchFiles, limit, setLimit, totalPages, activePage, setPage} = - useSearchFiles(24, emptySearch, mimetype, model); + filterOptions = filterOptions || [ + {key: 'video', text: 'Video', value: 'video'}, + {key: 'archive', text: 'Archive', value: 'archive'}, + {key: 'pdf', text: 'PDF', value: 'pdf'}, + {key: 'ebook', text: 'eBook', value: 'ebook'}, + {key: 'image', text: 'Image', value: 'image'}, + {key: 'zip', text: 'ZIP', value: 'zip'}, + ]; + + const {searchFiles, limit, setLimit, totalPages, activePage, setPage, filter} = + useSearchFiles(24, emptySearch, model); const {searchParams, updateQuery} = useQuery(); const setView = (value) => updateQuery({view: value}); const view = searchParams.get('view'); - const setMimetype = (value) => updateQuery({'mimetype': value}); - filterOptions = filterOptions || [ - {key: 'video', text: 'Video', value: 'video'}, - {key: 'archive', text: 'Archive', value: 'text/html'}, - {key: 'pdf', text: 'PDF', value: 'application/pdf'}, - {key: 'zip', text: 'ZIP', value: 'application/zip'}, - {key: 'image', text: 'Image', value: 'image'}, - ]; + const setFilter = (value) => updateQuery({'filter': value}); return } diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index ec555994..86f2b28c 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -25,7 +25,7 @@ import { } from "../api"; import {createSearchParams, useSearchParams} from "react-router-dom"; import {toast} from "react-semantic-toasts"; -import {enumerate, humanFileSize, secondsToFullDuration} from "../components/Common"; +import {enumerate, filterToMimetypes, humanFileSize, secondsToFullDuration} from "../components/Common"; import {StatusContext} from "../contexts/contexts"; const calculatePage = (offset, limit) => { @@ -386,12 +386,12 @@ export const useChannels = () => { return {channels, fetchChannels} } -export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype, model) => { +export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => { const {searchParams, updateQuery} = useQuery(); const limit = searchParams.get('l') || defaultLimit; const offset = searchParams.get('o') || 0; const searchStr = searchParams.get('q'); - const mimetype_ = searchParams.get('mimetype'); + const filter = searchParams.get('filter'); const model_ = searchParams.get('model'); const [searchFiles, setSearchFiles] = useState(null); @@ -402,11 +402,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype, if (!emptySearch && !searchStr) { return; } + const mimetypes = filterToMimetypes(filter); setSearchFiles(null); setTotalPages(0); try { let [files, total] = await filesSearch( - offset, limit, searchStr, mimetype || mimetype_, model || model_); + offset, limit, searchStr, mimetypes, model || model_); setSearchFiles(files); setTotalPages(calculateTotalPages(total, limit)); } catch (e) { @@ -422,7 +423,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype, useEffect(() => { localSearchFiles(); - }, [searchStr, limit, offset, activePage, mimetype, mimetype_, model, model_]); + }, [searchStr, limit, offset, activePage, filter, model, model_]); const setPage = (i) => { i = parseInt(i); @@ -440,7 +441,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype, updateQuery({l: value, o: 0}); } - return {searchFiles, totalPages, limit, searchStr, mimetype, setSearchStr, activePage, setPage, setLimit}; + return {searchFiles, totalPages, limit, searchStr, filter, setSearchStr, activePage, setPage, setLimit}; } export const useBrowseFiles = () => { diff --git a/nginx.conf b/nginx.conf index 33d7c876..1551adcb 100644 --- a/nginx.conf +++ b/nginx.conf @@ -10,7 +10,13 @@ http { '' close; } charset utf-8; + include mime.types; + types { + # Custom types for ebooks. + application/x-mobipocket-ebook mobi; + application/epub+zip epub; + } error_page 502 /error/50x.html; error_page 503 /error/50x.html; @@ -54,6 +60,7 @@ http { } location /media { + # The media files mounted in the docker volume sendfile on; sendfile_max_chunk 1m; diff --git a/nginx.dev.conf b/nginx.dev.conf index e2a7eaca..5ca4399b 100644 --- a/nginx.dev.conf +++ b/nginx.dev.conf @@ -2,13 +2,20 @@ events { } + http { map $http_upgrade $connection_upgrade { default upgrade; '' close; } charset utf-8; + include mime.types; + types { + # Custom types for ebooks. + application/x-mobipocket-ebook mobi; + application/epub+zip epub; + } server { location /favicon.ico { @@ -43,6 +50,7 @@ http { } location /media { + # The media files mounted in the docker volume sendfile on; sendfile_max_chunk 1m; diff --git a/requirements.txt b/requirements.txt index 528087eb..51265705 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ Pillow==9.3.0 Pint==0.18 +PyPDF2[crypto]==2.11.2 PyYAML==5.3.1 # this is the only version available on RPi SQLAlchemy==1.3.22 aiohttp==3.8.1 alembic==1.7.5 beautifulsoup4==4.10.0 cachetools==5.0.0 +ebooklib==0.18 feedparser==6.0.9 mock==4.0.3 psutil==5.9.2 diff --git a/test/example.epub b/test/example.epub new file mode 100644 index 00000000..8f65aece Binary files /dev/null and b/test/example.epub differ diff --git a/test/example.mobi b/test/example.mobi new file mode 100644 index 00000000..2c15b416 Binary files /dev/null and b/test/example.mobi differ diff --git a/test/example.pdf b/test/example.pdf new file mode 100644 index 00000000..bc2b9cd2 Binary files /dev/null and b/test/example.pdf differ diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 40685bb1..7c0939c1 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -231,6 +231,27 @@ def bad_vtt_file(test_directory) -> pathlib.Path: yield destination +@pytest.fixture +def example_pdf(test_directory) -> pathlib.Path: + destination = test_directory / 'example.pdf' + shutil.copy(PROJECT_DIR / 'test/example.pdf', destination) + yield destination + + +@pytest.fixture +def example_epub(test_directory) -> pathlib.Path: + destination = test_directory / 'example.epub' + shutil.copy(PROJECT_DIR / 'test/example.epub', destination) + yield destination + + +@pytest.fixture +def example_mobi(test_directory) -> pathlib.Path: + destination = test_directory / 'example.mobi' + shutil.copy(PROJECT_DIR / 'test/example.mobi', destination) + yield destination + + @pytest.fixture def make_files_structure(test_directory) -> Callable[[Union[List, Dict]], List[pathlib.Path]]: def create_files(paths: List) -> List[pathlib.Path]: @@ -277,7 +298,7 @@ def wrol_mode_fixture(test_config, test_download_manager): @pytest.fixture def mock_create_subprocess_shell(): - def mocker(communicate_return=None, return_code=None, communicate_side_effect = None): + def mocker(communicate_return=None, return_code=None, communicate_side_effect=None): async def communicate(*a, **kw): if communicate_side_effect: return communicate_side_effect() diff --git a/wrolpi/files/__init__.py b/wrolpi/files/__init__.py index 03476b2d..c6029615 100644 --- a/wrolpi/files/__init__.py +++ b/wrolpi/files/__init__.py @@ -1 +1,2 @@ +from . import ebooks # noqa from .api import bp diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 239bff60..889d520e 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -115,7 +115,7 @@ async def refresh_files_list(_: Request, body: schema.FilesRefreshListRequest): ) @validate(schema.FilesSearchRequest) async def post_search_files(_: Request, body: schema.FilesSearchRequest): - files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetype, body.model) + files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model) return json_response(dict(files=files, totals=dict(files=total))) diff --git a/wrolpi/files/ebooks.py b/wrolpi/files/ebooks.py new file mode 100644 index 00000000..43af4be3 --- /dev/null +++ b/wrolpi/files/ebooks.py @@ -0,0 +1,303 @@ +import dataclasses +import logging +import pathlib +from typing import Dict, List, Generator, Optional + +import bs4 +from sqlalchemy import Column, Integer, ForeignKey, String +from sqlalchemy.orm import Session, relationship + +from wrolpi.common import register_modeler, ModelHelper, Base, register_after_refresh +from wrolpi.db import get_db_session, get_db_curs +from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype +from wrolpi.files.models import File +from wrolpi.media_path import MediaPathType + +try: + import ebooklib + from ebooklib import epub +except ImportError: + ebooklib = None + epub = None + +logger = logging.getLogger(__name__) + +__all__ = [ + 'EBook', + 'EBookData', + 'extract_ebook_cover', + 'extract_ebook_data', + 'ebook_modeler', + 'MOBI_MIMETYPE', + 'EPUB_MIMETYPE' +] + +EBOOK_SUFFIXES = ('.epub', '.mobi') +EPUB_MIMETYPE = 'application/epub' +MOBI_MIMETYPE = 'application/x-mobipocket-ebook' + +EBOOK_MIMETYPES = ( + EPUB_MIMETYPE, + MOBI_MIMETYPE, +) + + +@dataclasses.dataclass +class EBookData: + cover: bytes = None + creator: str = None + text: str = None + title: str = None + + def __bool__(self): + return bool(self.cover) or bool(self.creator) or bool(self.text) or bool(self.title) + + +def extract_text(html: str) -> str: + soup = bs4.BeautifulSoup(html, features='html.parser') + text = soup.get_text() + return text + + +def mimetype_is_ebook(mimetype: str) -> bool: + return any(mimetype.startswith(i) for i in EBOOK_MIMETYPES) + + +def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]: + """Extract data from within an eBook file.""" + if ebooklib is None or epub is None: + raise ValueError('ebooklib is not installed') + if not mimetype.startswith(EPUB_MIMETYPE): + return None + + data = EBookData() + + book = epub.read_epub(path, options=dict(ignore_ncx=True)) + for key, value in book.metadata.items(): + if 'title' in value: + data.title = data.title or value['title'][0][0] + if 'creator' in value: + data.creator = data.creator or value['creator'][0][0] + + data.text = '' + for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT): + data.text += extract_text(doc.content) + + return data + + +def extract_ebook_cover(path: pathlib.Path) -> Optional[bytes]: + """Attempt to find the Cover within an ePub file. If no cover is found, return the first image.""" + if ebooklib is None or epub is None: + raise ValueError('ebooklib is not installed') + + book = epub.read_epub(path, options=dict(ignore_ncx=True)) + + for doc in book.get_items_of_type(ebooklib.ITEM_COVER): + logger.debug(f'Found cover for {path}') + return doc.content + + # Use the first image as the cover. + for doc in book.get_items_of_type(ebooklib.ITEM_IMAGE): + logger.debug(f'Using first image as cover for {path}') + return doc.content + + +def discover_calibre_cover(ebook_path: pathlib.Path): + """Calibre puts a cover file in a directory which contains only one ebook (but multiple formats of the ebook), + this function will return the path of the cover if the ebook_path is in such a situation.""" + if not ebook_path.is_file(): + raise ValueError(f'Invalid ebook path {ebook_path}') + + paths = list(ebook_path.parent.iterdir()) + + ebook_path_stem, _ = split_path_stem_and_suffix(ebook_path) + + cover = None + for path in paths: + stem, suffix = split_path_stem_and_suffix(path) + if path.is_dir(): + # We don't care about sub directories. + continue + if stem == ebook_path_stem: + # This is one of the formats of the ebook. + continue + if path.name == 'metadata.opf': + # Metadata, this is expected in a Calibre book directory. + continue + if stem == 'cover' and get_mimetype(path).startswith('image/'): + # Discovered the cover. + cover = path + continue + # Some file that does not match the ebook stem, metadata, or cover. This must not be a Calibre book directory. + return None + return cover + + +class EBook(ModelHelper, Base): + __tablename__ = 'ebook' + id = Column(Integer, primary_key=True) + size: int = Column(Integer) + + title: str = Column(String) + creator: str = Column(String) + + ebook_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE')) + ebook_file: File = relationship('File', primaryjoin='EBook.ebook_path==File.path') + cover_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE')) + cover_file: File = relationship('File', primaryjoin='EBook.cover_path==File.path') + + def __repr__(self): + cover = 'yes' if self.cover_file else 'no' + return f'' + + def my_paths(self) -> Generator[pathlib.Path, None, None]: + if self.ebook_path: + yield self.ebook_path + if self.cover_path: + yield self.cover_path + + def my_files(self) -> Generator[File, None, None]: + if self.ebook_file: + yield self.ebook_file + if self.cover_file: + yield self.cover_file + + def __json__(self): + d = dict( + cover_path=self.cover_path, + creator=self.creator, + ebook_path=self.ebook_path, + id=self.id, + size=self.size, + title=self.title, + ) + return d + + def generate_cover(self) -> Optional[pathlib.Path]: + """Discover this ebook's cover, if found, store it next to the ebook file.""" + path = self.ebook_file.path if self.ebook_file else self.ebook_path + + if not path: + raise ValueError('Cannot generate cover for ebook without ebook file!') + + if calibre_cover := discover_calibre_cover(path): + return calibre_cover + + cover_bytes = extract_ebook_cover(path) + if cover_bytes: + cover_path = path.with_suffix('.jpeg') + cover_path.write_bytes(cover_bytes) + return cover_path + + logger.warning(f'Unable to generate cover for {self.ebook_path}') + + @staticmethod + def find_by_path(path, session) -> Base: + return session.query(EBook).filter(EBook.ebook_path == path).one_or_none() + + +def model_ebook(session: Session, ebook_file: File, files: List[File]) -> EBook: + """ + Creates an EBook model based off a File. Searches for it's cover in the provided `files`. + """ + # Multiple formats may share this cover. + cover_file = next((i for i in files if i.mimetype.split('/')[0] == 'image'), None) + + ebook = session.query(EBook).filter_by(ebook_file=ebook_file).one_or_none() + if not ebook: + ebook = EBook(ebook_file=ebook_file) + session.add(ebook) + + size = ebook_file.path.stat().st_size + + ebook.ebook_file.do_index() + + index = size != ebook.size or cover_file != ebook.cover_file or not ebook.title + if not ebook.ebook_file.indexed or index: + try: + # Only read the contents of the file if it has changed. + data = extract_ebook_data(ebook_file.path, ebook_file.mimetype) + if data: + # Title is a_text. + ebook.ebook_file.a_text = ebook.title = data.title + # Creator is b_text. + ebook.ebook_file.b_text = ebook.creator = data.creator + # All text is d_text. + ebook.ebook_file.d_text = data.text + except Exception as e: + logger.error(f'Failed to extract book data', exc_info=e) + + if not ebook.title: + # Book was not indexed above (probably a MOBI), use the file data. + name, _ = split_path_stem_and_suffix(ebook.ebook_file.path) + ebook.title = ebook.ebook_file.a_text = name + + if cover_file: + cover_file.associated = True + + ebook.cover_file = cover_file + ebook.size = size + ebook_file.model = EBook.__tablename__ + + return ebook + + +@register_modeler +def ebook_modeler(groups: Dict[str, List[File]], session: Session): + """Searches for ebook files and models them into the ebook table. + + May generate cover files for EPUBs.""" + local_groups = groups.copy() + + for stem, group in local_groups.items(): + found_ebook = False + for file in group: + if not mimetype_is_ebook(file.mimetype): + continue + + session.flush(group) + model_ebook(session, file, group) + + found_ebook = True + + if found_ebook: + # Claim this group for this ebook. + del groups[stem] + + +@register_after_refresh +def discover_ebook_covers(): + """Discovers/extracts ebook covers for any Ebook that is missing one.""" + with get_db_curs() as curs: + stmt = f''' + SELECT e.id + FROM ebook e + LEFT JOIN file f on e.ebook_path = f.path + WHERE + e.cover_path IS NULL + AND f.mimetype LIKE '{EPUB_MIMETYPE}%' ''' + curs.execute(stmt) + missing_covers = [i[0] for i in curs.fetchall()] + + if not missing_covers: + logger.info('No ebook covers to generate') + return + + logger.warning(f'Discovering/generating {len(missing_covers)} ebook covers') + + for id_ in missing_covers: + with get_db_session(commit=True) as session: + ebook: EBook = session.query(EBook).filter(EBook.id == id_).one() + + cover_path = ebook.generate_cover() + if cover_path: + # Cover was generated or discovered. Associate it with the ebook. + cover_file = session.query(File).filter(File.path == cover_path).one_or_none() + if not cover_file: + cover_file = File(path=cover_path) + session.add(cover_file) + cover_file.do_index() + cover_file.associated = True + + ebook.cover_file = cover_file diff --git a/wrolpi/files/indexers.py b/wrolpi/files/indexers.py index 609840c7..8d9d2069 100644 --- a/wrolpi/files/indexers.py +++ b/wrolpi/files/indexers.py @@ -5,6 +5,11 @@ from collections import defaultdict from typing import List, Type, Tuple from zipfile import ZipFile +try: + from PyPDF2 import PdfReader +except ImportError: + PdfReader = None + from wrolpi.common import logger logger = logger.getChild(__name__) @@ -45,10 +50,10 @@ def find_indexer(file) -> Type[Indexer]: if not file.mimetype: return DefaultIndexer - if file.mimetype in indexer_map: - # Use the specific indexer. - indexer = indexer_map[file.mimetype] - else: + # Get the indexer that matches the mimetype of this file. + indexer = next((v for k, v in indexer_map.items() if file.mimetype.startswith(k)), None) + + if not indexer: # Use the broad indexer. indexer = indexer_map[file.mimetype.split('/')[0]] @@ -126,3 +131,36 @@ class TextIndexer(Indexer, ABC): # Some text files aren't for humans. return DefaultIndexer return cls + + +@register_indexer('application/pdf') +class PDFIndexer(Indexer, ABC): + """Uses PyPDF2 to extract text from a PDF.""" + + @classmethod + def create_index(cls, file): + path = file.path.path if hasattr(file.path, 'path') else file.path + a = cls.get_title(file) + try: + # PDFs are complex, don't fail to create title index if text extraction fails. + words = cls.get_words(path) + except Exception as e: + logger.error(f'Failed to index {path}', exc_info=e) + words = '' + return a, None, None, words + + @classmethod + def get_words(cls, path: pathlib.Path) -> str: + if PdfReader is None: + return '' + + reader = PdfReader(path) + words = [] + for page in reader.pages: + # Get all text facing up. + text = page.extract_text(0) + words.append(text) + words = '\n'.join(words) + # Postgres does not allow null characters. + words = words.replace('\x00', '\uFFFD') + return words diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 91105548..3a520e2a 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -28,11 +28,10 @@ try: import magic mime = magic.Magic(mime=True) - - no_magic = False except ImportError: # Magic is not installed - no_magic = True + magic = None + mime = None logger = logger.getChild(__name__) @@ -95,9 +94,20 @@ FILE_NAME_REGEX = re.compile(r'[_ .]') FILE_BIN = which('file', '/usr/bin/file') +def _mimetype_suffix_map(path: Path, mimetype: str): + """Special handling for mimetypes. Python Magic may not return the correct mimetype.""" + from wrolpi.files.ebooks import MOBI_MIMETYPE + suffix = path.suffix.lower() + if mimetype == 'application/octet-stream' and suffix.endswith('.mobi'): + return MOBI_MIMETYPE + if mimetype == 'text/plain' and suffix.endswith('.json'): + return 'application/json' + return mimetype + + def get_mimetype(path: Path) -> str: """Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command.""" - if no_magic: + if magic is None: # This method is slow, prefer the speedier `magic` module. cmd = (FILE_BIN, '--mime-type', str(path.absolute())) output = subprocess.check_output(cmd) @@ -105,7 +115,9 @@ def get_mimetype(path: Path) -> str: mimetype = output.split(' ')[-1].strip() return mimetype else: - return mime.from_file(path) + mimetype = mime.from_file(path) + mimetype = _mimetype_suffix_map(path, mimetype) + return mimetype # Special suffixes within WROLPi. @@ -328,17 +340,18 @@ async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str refresh_logger.info(f'Done refreshing files in {directory}') -def search_files(search_str: str, limit: int, offset: int, mimetype: str = None, model: str = None) -> \ +def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None) -> \ Tuple[List[dict], int]: - """ - Search the Files table. Order the returned Files by their rank if `search_str` is provided. Return all files if + """Search the Files table. + + Order the returned Files by their rank if `search_str` is provided. Return all files if `search_str` is empty. Parameters: search_str: Search the ts_vector of the file. Returns all files if this is empty. limit: Return only this many files. offset: Offset the query. - mimetype: Only return files that match this mimetype. + mimetypes: Only return files that match these mimetypes. model: Only return files that match this model. """ params = dict(offset=offset, limit=limit) @@ -352,12 +365,17 @@ def search_files(search_str: str, limit: int, offset: int, mimetype: str = None, selects.append('ts_rank(textsearch, websearch_to_tsquery(%(search_str)s))') order_by = '2 DESC' - if mimetype and len(mimetype.split('/')) == 1: - params['mimetype'] = f'{mimetype}/%' - wheres.append('mimetype LIKE %(mimetype)s') - elif mimetype: - wheres.append('mimetype = %(mimetype)s') - params['mimetype'] = mimetype + if mimetypes: + mimetype_wheres = [] + for idx, mimetype in enumerate(mimetypes): + key = f'mimetype{idx}' + if len(mimetype.split('/')) == 1: + params[key] = f'{mimetype}/%' + else: + params[key] = f'{mimetype}%' + mimetype_wheres.append(f'mimetype LIKE %({key})s') + mimetype_wheres = ' OR '.join(mimetype_wheres) + wheres.append(f'({mimetype_wheres})') if model: params['model'] = model diff --git a/wrolpi/files/schema.py b/wrolpi/files/schema.py index 93a3adac..b7d04c3d 100644 --- a/wrolpi/files/schema.py +++ b/wrolpi/files/schema.py @@ -17,7 +17,7 @@ class FilesSearchRequest: search_str: Optional[str] = None limit: Optional[int] = 20 offset: Optional[int] = 0 - mimetype: Optional[str] = None + mimetypes: List[str] = field(default_factory=list) model: Optional[str] = None diff --git a/wrolpi/files/test/test_ebooks.py b/wrolpi/files/test/test_ebooks.py new file mode 100644 index 00000000..5180b566 --- /dev/null +++ b/wrolpi/files/test/test_ebooks.py @@ -0,0 +1,127 @@ +import json +import shutil +from http import HTTPStatus + +import pytest + +from wrolpi.files.ebooks import EBook, MOBI_MIMETYPE, EPUB_MIMETYPE +from wrolpi.files.lib import refresh_files +from wrolpi.test.common import assert_dict_contains + + +@pytest.mark.asyncio +async def test_index(test_session, test_directory, example_epub, example_mobi): + """Ebooks can be indexed. + + Covers can be discovered.""" + await refresh_files() + + ebook_epub, ebook_mobi = test_session.query(EBook).order_by(EBook.ebook_path) + ebook_epub: EBook + ebook_mobi: EBook + + assert ebook_epub.title == 'WROLPi Test Book' + assert ebook_epub.ebook_file.mimetype.startswith(EPUB_MIMETYPE) + assert ebook_epub.creator == 'roland' + assert ebook_epub.size == ebook_epub.ebook_file.size + assert ebook_epub.cover_path and ebook_epub.cover_path.is_file() + # The cover is the WROLPi logo. + assert ebook_epub.cover_file.size == 297099 + + assert ebook_epub.ebook_file.a_text, 'Book title was not indexed' + assert ebook_epub.ebook_file.b_text, 'Book creator was not indexed' + assert ebook_epub.ebook_file.d_text, 'Book text was not indexed' + + # Mobi is not fully supported, title is the file name. No creator or cover. + assert ebook_mobi.title == 'example' + assert ebook_mobi.ebook_file.mimetype == MOBI_MIMETYPE + assert not ebook_mobi.creator + assert not ebook_mobi.cover_path + + # Ebooks can be deleted during refresh. + example_mobi.unlink() + example_epub.unlink() + await refresh_files() + assert test_session.query(EBook).count() == 0 + + +def test_search(test_session, test_client, example_epub): + """Ebooks are handled in File search results.""" + request, response = test_client.post('/api/files/refresh') + assert response.status == HTTPStatus.NO_CONTENT + + assert test_session.query(EBook).count() == 1 + + ebook: EBook = test_session.query(EBook).one() + assert ebook.title == 'WROLPi Test Book' + assert ebook.creator == 'roland' + assert ebook.ebook_file.a_text == 'WROLPi Test Book', 'Book title was not updated' + assert ebook.ebook_file.d_text, 'Book was not indexed' + + content = dict(mimetypes=['application/epub', 'application/x-mobipocket-ebook']) + request, response = test_client.post('/api/files/search', content=json.dumps(content)) + assert response.status == HTTPStatus.OK + assert response.json + result = response.json['files'][0] + assert result['path'] == 'example.epub' + assert result['mimetype'] == 'application/epub+zip' + assert_dict_contains( + result['ebook'], + {'cover_path': 'example.jpeg', 'ebook_path': 'example.epub'}, + ) + + content = dict(mimetypes=['application/epub+zip', 'application/x-mobipocket-ebook']) + request, response = test_client.post('/api/files/search', content=json.dumps(content)) + result = response.json['files'][0] + assert result['path'] == 'example.epub' + assert result['mimetype'] == 'application/epub+zip' + + content = dict(mimetypes=[]) + request, response = test_client.post('/api/files/search', content=json.dumps(content)) + assert response.status == HTTPStatus.OK + assert response.json + + +@pytest.mark.asyncio +async def test_discover_calibre_cover(test_session, test_directory, example_epub, image_file): + """Calibre puts a cover near an ebook file, test if it can be found.""" + # Create two files in the + metadata = test_directory / 'metadata.opf' + metadata.touch() + # Create an alternate format next to the epub. This will be ignored because it is not yet supported. + mobi_path = example_epub.with_suffix('.mobi') + mobi_path.touch() + + await refresh_files() + + assert test_session.query(EBook).count() == 1 + ebook: EBook = test_session.query(EBook).one() + assert ebook.title == 'WROLPi Test Book' + # The cover in the ebook was extracted. + assert ebook.cover_file.size == 297099 + + # Delete the extracted cover, use the cover image from Calibre. + ebook.cover_path.unlink() + test_session.delete(ebook.cover_file) + cover_image = test_directory / 'cover.jpg' + shutil.move(image_file, cover_image) + + # Calibre cover image was discovered, no cover was generated. + await refresh_files() + ebook: EBook = test_session.query(EBook).one() + assert ebook.title == 'WROLPi Test Book' + assert ebook.cover_file.size == 641 + + # Create a file nearby, its no longer a Calibre directory. + mobi_path = mobi_path.rename(test_directory / 'other book.mobi') + await refresh_files() + assert ebook.title == 'WROLPi Test Book' + assert ebook.cover_file.size == 297099 + mobi_path.unlink() + + # Delete the discovered cover, cover should be generated. + cover_image.unlink() + await refresh_files() + ebook: EBook = test_session.query(EBook).one() + assert ebook.title == 'WROLPi Test Book' + assert ebook.cover_file.size == 297099 diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 5deb9886..6181dfb6 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -469,3 +469,25 @@ def test_matching_directories(make_files_structure, test_directory): # foo is an exact match, return subdirectories matches = lib.get_matching_directories(test_directory / 'foo') assert matches == [str(test_directory / 'foo/qux')] + + +def test_pdf_indexer(example_pdf): + """PDFs can be indexed by PDFIndexer.""" + file = File(path=example_pdf) + + a_text, b_text, c_text, d_text = indexers.PDFIndexer.create_index(file) + + # The file name. + assert a_text == ('example', '.pdf') + assert b_text is None + assert c_text is None + # Both pages of the example PDF are extracted. + assert d_text == 'Page one\nPage two' + + +def test_get_mimetype(example_epub, example_mobi, example_pdf, image_file, video_file): + assert lib.get_mimetype(example_epub) == 'application/epub+zip' + assert lib.get_mimetype(example_mobi) == 'application/x-mobipocket-ebook' + assert lib.get_mimetype(example_pdf) == 'application/pdf' + assert lib.get_mimetype(image_file) == 'image/jpeg' + assert lib.get_mimetype(video_file) == 'video/mp4'
{ebook ? humanFileSize(ebook.size) : null}
{suffix}