mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Storing example ebooks for testing.
Special handling of json mimetype for json files. Moving types up to avoid overwriting mime.types. Adding custom types to nginx to server ebooks better. Displaying EBookCard correctly when ebook was not indexed correctly. Global search can search for all ebooks (mobi/epub). Testing when mimetype suffix is searched. Adding eBook to global search filter list. Opening mimetype search to match mimetypes that may not end with expected suffix. Adding log warning to explain refresh speed. Generating ebook covers after refresh. Testing that another nearby book removes the cover. Calibre covers can be discovered and associated with an ebook. Handling if ebooklib is not yet installed. Adding ebook table and modeler to allow searching the contents of epub files. Marking all PDFs as un-indexed, so they will be indexed. Replacing null characters in PDF text. Testing PDFIndexer against example PDF file. Installing PyPDF2 to extract text for PDFIndexer.
This commit is contained in:
parent
d77e69f54e
commit
6fcb992540
21 changed files with 769 additions and 52 deletions
66
alembic/versions/717f36a4da94_ebook.py
Normal file
66
alembic/versions/717f36a4da94_ebook.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
"""Ebook
|
||||
|
||||
Revision ID: 717f36a4da94
|
||||
Revises: d0bbe45e56c3
|
||||
Create Date: 2022-12-09 15:32:32.228539
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '717f36a4da94'
|
||||
down_revision = 'd0bbe45e56c3'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('''
|
||||
DROP INDEX IF EXISTS ebook_title_idx;
|
||||
DROP INDEX IF EXISTS ebook_size_idx;
|
||||
DROP INDEX IF EXISTS ebook_creator_idx;
|
||||
DROP INDEX IF EXISTS ebook_ebook_path_idx;
|
||||
DROP INDEX IF EXISTS ebook_cover_path_idx;
|
||||
DROP TABLE IF EXISTS ebook;
|
||||
''')
|
||||
|
||||
session.execute('''
|
||||
CREATE TABLE ebook (
|
||||
id SERIAL PRIMARY KEY,
|
||||
size INTEGER,
|
||||
title TEXT,
|
||||
creator TEXT,
|
||||
ebook_path TEXT REFERENCES file(path) ON DELETE CASCADE,
|
||||
cover_path TEXT REFERENCES file(path) ON DELETE CASCADE
|
||||
)''')
|
||||
|
||||
if not DOCKERIZED:
|
||||
session.execute('ALTER TABLE public.ebook OWNER TO wrolpi')
|
||||
|
||||
session.execute('CREATE INDEX ebook_title_idx ON ebook(title)')
|
||||
session.execute('CREATE INDEX ebook_size_idx ON ebook(size)')
|
||||
session.execute('CREATE INDEX ebook_creator_idx ON ebook(creator)')
|
||||
session.execute('CREATE INDEX ebook_ebook_path_idx ON ebook(ebook_path)')
|
||||
session.execute('CREATE INDEX ebook_cover_path_idx ON ebook(cover_path)')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('''
|
||||
DROP INDEX IF EXISTS ebook_title_idx;
|
||||
DROP INDEX IF EXISTS ebook_size_idx;
|
||||
DROP INDEX IF EXISTS ebook_creator_idx;
|
||||
DROP INDEX IF EXISTS ebook_ebook_path_idx;
|
||||
DROP INDEX IF EXISTS ebook_cover_path_idx;
|
||||
DROP TABLE IF EXISTS ebook;
|
||||
''')
|
||||
30
alembic/versions/d0bbe45e56c3_reindex_pdfs.py
Normal file
30
alembic/versions/d0bbe45e56c3_reindex_pdfs.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
"""Reindex pdfs
|
||||
|
||||
Revision ID: d0bbe45e56c3
|
||||
Revises: dbf05c8e8dcc
|
||||
Create Date: 2022-12-09 12:27:18.908533
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'd0bbe45e56c3'
|
||||
down_revision = 'dbf05c8e8dcc'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute("UPDATE file SET indexed=false WHERE mimetype = 'application/pdf'")
|
||||
|
||||
|
||||
def downgrade():
|
||||
pass
|
||||
|
|
@ -484,10 +484,10 @@ const replaceFileDatetimes = (files) => {
|
|||
return files;
|
||||
}
|
||||
|
||||
export async function filesSearch(offset, limit, searchStr, mimetype, model) {
|
||||
export async function filesSearch(offset, limit, searchStr, mimetypes, model) {
|
||||
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
|
||||
if (mimetype) {
|
||||
body['mimetype'] = mimetype;
|
||||
if (mimetypes) {
|
||||
body['mimetypes'] = mimetypes;
|
||||
}
|
||||
if (model) {
|
||||
body['model'] = model;
|
||||
|
|
|
|||
|
|
@ -423,16 +423,24 @@ export function CardGroupCentered(props) {
|
|||
</div>)
|
||||
}
|
||||
|
||||
export function CardPosterLink({to, poster_url, imageLabel}) {
|
||||
export function CardPosterLink({to, poster_url, imageLabel, external = false}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
const style = {display: 'flex', justifyContent: 'center', ...s['style']};
|
||||
return <Link to={to} style={style}>
|
||||
<Image fluid
|
||||
src={poster_url}
|
||||
label={imageLabel}
|
||||
style={{maxHeight: '163px', width: 'auto'}}
|
||||
/>
|
||||
</Link>
|
||||
const image = <Image fluid
|
||||
src={poster_url}
|
||||
label={imageLabel}
|
||||
style={{maxHeight: '163px', width: 'auto'}}
|
||||
/>;
|
||||
if (external === true) {
|
||||
return <a href={to} target='_blank' style={style}>
|
||||
{image}
|
||||
</a>
|
||||
} else {
|
||||
// Link using React Router.
|
||||
return <Link to={to} style={style}>
|
||||
{image}
|
||||
</Link>
|
||||
}
|
||||
}
|
||||
|
||||
export function HelpPopup({icon, size, content, position}) {
|
||||
|
|
@ -567,6 +575,9 @@ export function mimetypeColor(mimetype) {
|
|||
return 'green'
|
||||
} else if (mimetype && mimetype.startsWith('application/zip')) {
|
||||
return 'purple'
|
||||
} else if (mimetype && (
|
||||
mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook'))) {
|
||||
return 'yellow'
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
|
|
@ -595,6 +606,8 @@ export function FileIcon({file, disabled = true, size = 'huge', ...props}) {
|
|||
props['name'] = 'file archive';
|
||||
} else if (mimetype.startsWith('application/x-iso9660-image')) {
|
||||
props['name'] = 'dot circle';
|
||||
} else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook')) {
|
||||
props['name'] = 'book';
|
||||
}
|
||||
}
|
||||
return <Icon disabled={disabled} size={size} {...props}/>
|
||||
|
|
@ -763,3 +776,19 @@ export const ColorToSemanticHexColor = (color) => {
|
|||
}
|
||||
return colorMap[color] || null;
|
||||
}
|
||||
|
||||
export const filterToMimetypes = (filter) => {
|
||||
if (filter === 'video') {
|
||||
return ['video'];
|
||||
} else if (filter === 'archive') {
|
||||
return ['text/html'];
|
||||
} else if (filter === 'pdf') {
|
||||
return ['application/pdf'];
|
||||
} else if (filter === 'ebook') {
|
||||
return ['application/epub+zip', 'application/x-mobipocket-ebook'];
|
||||
} else if (filter === 'image') {
|
||||
return ['image'];
|
||||
} else if (filter === 'zip') {
|
||||
return ['application/zip'];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
CardHeader,
|
||||
CardMeta,
|
||||
Confirm,
|
||||
Container,
|
||||
Divider,
|
||||
Dropdown,
|
||||
Image,
|
||||
|
|
@ -17,6 +18,8 @@ import 'react-keyed-file-browser/dist/react-keyed-file-browser.css';
|
|||
import {deleteFile, refreshDirectoryFiles, refreshFiles} from "../api";
|
||||
import {
|
||||
CardGroupCentered,
|
||||
CardPosterLink,
|
||||
epochToDateString,
|
||||
ExternalCardLink,
|
||||
FileIcon,
|
||||
humanFileSize,
|
||||
|
|
@ -25,7 +28,6 @@ import {
|
|||
PageContainer,
|
||||
Paginator,
|
||||
textEllipsis,
|
||||
epochToDateString,
|
||||
useTitle
|
||||
} from "./Common";
|
||||
import {useBrowseFiles, useQuery, useSearchFiles} from "../hooks/customHooks";
|
||||
|
|
@ -37,6 +39,7 @@ import {ThemeContext} from "../contexts/contexts";
|
|||
import {Button, Card, CardIcon, Header, Icon, Placeholder, Segment} from "./Theme";
|
||||
import {SelectableTable} from "./Tables";
|
||||
import {VideoCard, VideoRowCells} from "./Videos";
|
||||
import _ from 'lodash';
|
||||
|
||||
const icons = {
|
||||
File: <Icon name='file'/>,
|
||||
|
|
@ -142,6 +145,42 @@ export function Files() {
|
|||
</>
|
||||
}
|
||||
|
||||
function EbookCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
const url = `/media/${encodeURIComponent(file.path)}`;
|
||||
let {ebook, suffix} = file;
|
||||
const coverSrc = ebook && ebook.cover_path ? `/media/${encodeURIComponent(ebook.cover_path)}` : null;
|
||||
|
||||
let cover = <CardIcon><FileIcon file={file}/></CardIcon>;
|
||||
if (coverSrc) {
|
||||
cover = <CardPosterLink to={url} poster_url={coverSrc} external={true}/>;
|
||||
}
|
||||
|
||||
suffix = suffix ? _.trimStart(suffix, '.').toUpperCase() : null;
|
||||
|
||||
const color = mimetypeColor(file.mimetype);
|
||||
return <Card color={color}>
|
||||
{cover}
|
||||
<CardContent {...s}>
|
||||
<CardHeader>
|
||||
<Container textAlign='left'>
|
||||
<ExternalCardLink to={url}>{ebook ? ebook.title : file.title}</ExternalCardLink>
|
||||
</Container>
|
||||
</CardHeader>
|
||||
<CardDescription>
|
||||
<Container textAlign='left'>
|
||||
<b {...s}>{ebook ? ebook.creator : null}</b>
|
||||
</Container>
|
||||
</CardDescription>
|
||||
<CardMeta>
|
||||
<p {...s}>{ebook ? humanFileSize(ebook.size) : null}</p>
|
||||
<pre {...s}>{suffix}</pre>
|
||||
</CardMeta>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
}
|
||||
|
||||
function ImageCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
const url = `/media/${encodeURIComponent(file.path)}`;
|
||||
|
|
@ -184,6 +223,10 @@ function FileCard({file}) {
|
|||
return <ArchiveCard key={file['path']} file={file}/>;
|
||||
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
|
||||
return <ImageCard key={file['path']} file={file}/>;
|
||||
} else if (file.mimetype && (
|
||||
file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
|
||||
)) {
|
||||
return <EbookCard key={file['path']} file={file}/>;
|
||||
}
|
||||
|
||||
const url = `/media/${encodeURIComponent(file.path)}`;
|
||||
|
|
@ -399,26 +442,27 @@ export function FilesSearchView({
|
|||
showSelect = false,
|
||||
emptySearch = false,
|
||||
filterOptions,
|
||||
mimetype,
|
||||
model,
|
||||
onSelect,
|
||||
setFilters,
|
||||
}) {
|
||||
const {searchFiles, limit, setLimit, totalPages, activePage, setPage} =
|
||||
useSearchFiles(24, emptySearch, mimetype, model);
|
||||
filterOptions = filterOptions || [
|
||||
{key: 'video', text: 'Video', value: 'video'},
|
||||
{key: 'archive', text: 'Archive', value: 'archive'},
|
||||
{key: 'pdf', text: 'PDF', value: 'pdf'},
|
||||
{key: 'ebook', text: 'eBook', value: 'ebook'},
|
||||
{key: 'image', text: 'Image', value: 'image'},
|
||||
{key: 'zip', text: 'ZIP', value: 'zip'},
|
||||
];
|
||||
|
||||
const {searchFiles, limit, setLimit, totalPages, activePage, setPage, filter} =
|
||||
useSearchFiles(24, emptySearch, model);
|
||||
|
||||
const {searchParams, updateQuery} = useQuery();
|
||||
const setView = (value) => updateQuery({view: value});
|
||||
const view = searchParams.get('view');
|
||||
|
||||
const setMimetype = (value) => updateQuery({'mimetype': value});
|
||||
filterOptions = filterOptions || [
|
||||
{key: 'video', text: 'Video', value: 'video'},
|
||||
{key: 'archive', text: 'Archive', value: 'text/html'},
|
||||
{key: 'pdf', text: 'PDF', value: 'application/pdf'},
|
||||
{key: 'zip', text: 'ZIP', value: 'application/zip'},
|
||||
{key: 'image', text: 'Image', value: 'image'},
|
||||
];
|
||||
const setFilter = (value) => updateQuery({'filter': value});
|
||||
|
||||
return <FilesView
|
||||
files={searchFiles}
|
||||
|
|
@ -434,7 +478,7 @@ export function FilesSearchView({
|
|||
onSelect={onSelect}
|
||||
setPage={setPage}
|
||||
filterOptions={filterOptions}
|
||||
setFilters={setFilters || setMimetype}
|
||||
setFilters={setFilters || setFilter}
|
||||
/>
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import {
|
|||
} from "../api";
|
||||
import {createSearchParams, useSearchParams} from "react-router-dom";
|
||||
import {toast} from "react-semantic-toasts";
|
||||
import {enumerate, humanFileSize, secondsToFullDuration} from "../components/Common";
|
||||
import {enumerate, filterToMimetypes, humanFileSize, secondsToFullDuration} from "../components/Common";
|
||||
import {StatusContext} from "../contexts/contexts";
|
||||
|
||||
const calculatePage = (offset, limit) => {
|
||||
|
|
@ -386,12 +386,12 @@ export const useChannels = () => {
|
|||
return {channels, fetchChannels}
|
||||
}
|
||||
|
||||
export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype, model) => {
|
||||
export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => {
|
||||
const {searchParams, updateQuery} = useQuery();
|
||||
const limit = searchParams.get('l') || defaultLimit;
|
||||
const offset = searchParams.get('o') || 0;
|
||||
const searchStr = searchParams.get('q');
|
||||
const mimetype_ = searchParams.get('mimetype');
|
||||
const filter = searchParams.get('filter');
|
||||
const model_ = searchParams.get('model');
|
||||
|
||||
const [searchFiles, setSearchFiles] = useState(null);
|
||||
|
|
@ -402,11 +402,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype,
|
|||
if (!emptySearch && !searchStr) {
|
||||
return;
|
||||
}
|
||||
const mimetypes = filterToMimetypes(filter);
|
||||
setSearchFiles(null);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [files, total] = await filesSearch(
|
||||
offset, limit, searchStr, mimetype || mimetype_, model || model_);
|
||||
offset, limit, searchStr, mimetypes, model || model_);
|
||||
setSearchFiles(files);
|
||||
setTotalPages(calculateTotalPages(total, limit));
|
||||
} catch (e) {
|
||||
|
|
@ -422,7 +423,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype,
|
|||
|
||||
useEffect(() => {
|
||||
localSearchFiles();
|
||||
}, [searchStr, limit, offset, activePage, mimetype, mimetype_, model, model_]);
|
||||
}, [searchStr, limit, offset, activePage, filter, model, model_]);
|
||||
|
||||
const setPage = (i) => {
|
||||
i = parseInt(i);
|
||||
|
|
@ -440,7 +441,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, mimetype,
|
|||
updateQuery({l: value, o: 0});
|
||||
}
|
||||
|
||||
return {searchFiles, totalPages, limit, searchStr, mimetype, setSearchStr, activePage, setPage, setLimit};
|
||||
return {searchFiles, totalPages, limit, searchStr, filter, setSearchStr, activePage, setPage, setLimit};
|
||||
}
|
||||
|
||||
export const useBrowseFiles = () => {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,13 @@ http {
|
|||
'' close;
|
||||
}
|
||||
charset utf-8;
|
||||
|
||||
include mime.types;
|
||||
types {
|
||||
# Custom types for ebooks.
|
||||
application/x-mobipocket-ebook mobi;
|
||||
application/epub+zip epub;
|
||||
}
|
||||
|
||||
error_page 502 /error/50x.html;
|
||||
error_page 503 /error/50x.html;
|
||||
|
|
@ -54,6 +60,7 @@ http {
|
|||
}
|
||||
|
||||
location /media {
|
||||
|
||||
# The media files mounted in the docker volume
|
||||
sendfile on;
|
||||
sendfile_max_chunk 1m;
|
||||
|
|
|
|||
|
|
@ -2,13 +2,20 @@ events {
|
|||
|
||||
}
|
||||
|
||||
|
||||
http {
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
charset utf-8;
|
||||
|
||||
include mime.types;
|
||||
types {
|
||||
# Custom types for ebooks.
|
||||
application/x-mobipocket-ebook mobi;
|
||||
application/epub+zip epub;
|
||||
}
|
||||
|
||||
server {
|
||||
location /favicon.ico {
|
||||
|
|
@ -43,6 +50,7 @@ http {
|
|||
}
|
||||
|
||||
location /media {
|
||||
|
||||
# The media files mounted in the docker volume
|
||||
sendfile on;
|
||||
sendfile_max_chunk 1m;
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
Pillow==9.3.0
|
||||
Pint==0.18
|
||||
PyPDF2[crypto]==2.11.2
|
||||
PyYAML==5.3.1 # this is the only version available on RPi
|
||||
SQLAlchemy==1.3.22
|
||||
aiohttp==3.8.1
|
||||
alembic==1.7.5
|
||||
beautifulsoup4==4.10.0
|
||||
cachetools==5.0.0
|
||||
ebooklib==0.18
|
||||
feedparser==6.0.9
|
||||
mock==4.0.3
|
||||
psutil==5.9.2
|
||||
|
|
|
|||
BIN
test/example.epub
Normal file
BIN
test/example.epub
Normal file
Binary file not shown.
BIN
test/example.mobi
Normal file
BIN
test/example.mobi
Normal file
Binary file not shown.
BIN
test/example.pdf
Normal file
BIN
test/example.pdf
Normal file
Binary file not shown.
|
|
@ -231,6 +231,27 @@ def bad_vtt_file(test_directory) -> pathlib.Path:
|
|||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_pdf(test_directory) -> pathlib.Path:
|
||||
destination = test_directory / 'example.pdf'
|
||||
shutil.copy(PROJECT_DIR / 'test/example.pdf', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_epub(test_directory) -> pathlib.Path:
|
||||
destination = test_directory / 'example.epub'
|
||||
shutil.copy(PROJECT_DIR / 'test/example.epub', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_mobi(test_directory) -> pathlib.Path:
|
||||
destination = test_directory / 'example.mobi'
|
||||
shutil.copy(PROJECT_DIR / 'test/example.mobi', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_files_structure(test_directory) -> Callable[[Union[List, Dict]], List[pathlib.Path]]:
|
||||
def create_files(paths: List) -> List[pathlib.Path]:
|
||||
|
|
@ -277,7 +298,7 @@ def wrol_mode_fixture(test_config, test_download_manager):
|
|||
|
||||
@pytest.fixture
|
||||
def mock_create_subprocess_shell():
|
||||
def mocker(communicate_return=None, return_code=None, communicate_side_effect = None):
|
||||
def mocker(communicate_return=None, return_code=None, communicate_side_effect=None):
|
||||
async def communicate(*a, **kw):
|
||||
if communicate_side_effect:
|
||||
return communicate_side_effect()
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
from . import ebooks # noqa
|
||||
from .api import bp
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ async def refresh_files_list(_: Request, body: schema.FilesRefreshListRequest):
|
|||
)
|
||||
@validate(schema.FilesSearchRequest)
|
||||
async def post_search_files(_: Request, body: schema.FilesSearchRequest):
|
||||
files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetype, body.model)
|
||||
files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model)
|
||||
return json_response(dict(files=files, totals=dict(files=total)))
|
||||
|
||||
|
||||
|
|
|
|||
303
wrolpi/files/ebooks.py
Normal file
303
wrolpi/files/ebooks.py
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
import dataclasses
|
||||
import logging
|
||||
import pathlib
|
||||
from typing import Dict, List, Generator, Optional
|
||||
|
||||
import bs4
|
||||
from sqlalchemy import Column, Integer, ForeignKey, String
|
||||
from sqlalchemy.orm import Session, relationship
|
||||
|
||||
from wrolpi.common import register_modeler, ModelHelper, Base, register_after_refresh
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
||||
try:
|
||||
import ebooklib
|
||||
from ebooklib import epub
|
||||
except ImportError:
|
||||
ebooklib = None
|
||||
epub = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
'EBook',
|
||||
'EBookData',
|
||||
'extract_ebook_cover',
|
||||
'extract_ebook_data',
|
||||
'ebook_modeler',
|
||||
'MOBI_MIMETYPE',
|
||||
'EPUB_MIMETYPE'
|
||||
]
|
||||
|
||||
EBOOK_SUFFIXES = ('.epub', '.mobi')
|
||||
EPUB_MIMETYPE = 'application/epub'
|
||||
MOBI_MIMETYPE = 'application/x-mobipocket-ebook'
|
||||
|
||||
EBOOK_MIMETYPES = (
|
||||
EPUB_MIMETYPE,
|
||||
MOBI_MIMETYPE,
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class EBookData:
|
||||
cover: bytes = None
|
||||
creator: str = None
|
||||
text: str = None
|
||||
title: str = None
|
||||
|
||||
def __bool__(self):
|
||||
return bool(self.cover) or bool(self.creator) or bool(self.text) or bool(self.title)
|
||||
|
||||
|
||||
def extract_text(html: str) -> str:
|
||||
soup = bs4.BeautifulSoup(html, features='html.parser')
|
||||
text = soup.get_text()
|
||||
return text
|
||||
|
||||
|
||||
def mimetype_is_ebook(mimetype: str) -> bool:
|
||||
return any(mimetype.startswith(i) for i in EBOOK_MIMETYPES)
|
||||
|
||||
|
||||
def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]:
|
||||
"""Extract data from within an eBook file."""
|
||||
if ebooklib is None or epub is None:
|
||||
raise ValueError('ebooklib is not installed')
|
||||
if not mimetype.startswith(EPUB_MIMETYPE):
|
||||
return None
|
||||
|
||||
data = EBookData()
|
||||
|
||||
book = epub.read_epub(path, options=dict(ignore_ncx=True))
|
||||
for key, value in book.metadata.items():
|
||||
if 'title' in value:
|
||||
data.title = data.title or value['title'][0][0]
|
||||
if 'creator' in value:
|
||||
data.creator = data.creator or value['creator'][0][0]
|
||||
|
||||
data.text = ''
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||||
data.text += extract_text(doc.content)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def extract_ebook_cover(path: pathlib.Path) -> Optional[bytes]:
|
||||
"""Attempt to find the Cover within an ePub file. If no cover is found, return the first image."""
|
||||
if ebooklib is None or epub is None:
|
||||
raise ValueError('ebooklib is not installed')
|
||||
|
||||
book = epub.read_epub(path, options=dict(ignore_ncx=True))
|
||||
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_COVER):
|
||||
logger.debug(f'Found cover for {path}')
|
||||
return doc.content
|
||||
|
||||
# Use the first image as the cover.
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_IMAGE):
|
||||
logger.debug(f'Using first image as cover for {path}')
|
||||
return doc.content
|
||||
|
||||
|
||||
def discover_calibre_cover(ebook_path: pathlib.Path):
|
||||
"""Calibre puts a cover file in a directory which contains only one ebook (but multiple formats of the ebook),
|
||||
this function will return the path of the cover if the ebook_path is in such a situation."""
|
||||
if not ebook_path.is_file():
|
||||
raise ValueError(f'Invalid ebook path {ebook_path}')
|
||||
|
||||
paths = list(ebook_path.parent.iterdir())
|
||||
|
||||
ebook_path_stem, _ = split_path_stem_and_suffix(ebook_path)
|
||||
|
||||
cover = None
|
||||
for path in paths:
|
||||
stem, suffix = split_path_stem_and_suffix(path)
|
||||
if path.is_dir():
|
||||
# We don't care about sub directories.
|
||||
continue
|
||||
if stem == ebook_path_stem:
|
||||
# This is one of the formats of the ebook.
|
||||
continue
|
||||
if path.name == 'metadata.opf':
|
||||
# Metadata, this is expected in a Calibre book directory.
|
||||
continue
|
||||
if stem == 'cover' and get_mimetype(path).startswith('image/'):
|
||||
# Discovered the cover.
|
||||
cover = path
|
||||
continue
|
||||
# Some file that does not match the ebook stem, metadata, or cover. This must not be a Calibre book directory.
|
||||
return None
|
||||
return cover
|
||||
|
||||
|
||||
class EBook(ModelHelper, Base):
|
||||
__tablename__ = 'ebook'
|
||||
id = Column(Integer, primary_key=True)
|
||||
size: int = Column(Integer)
|
||||
|
||||
title: str = Column(String)
|
||||
creator: str = Column(String)
|
||||
|
||||
ebook_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
ebook_file: File = relationship('File', primaryjoin='EBook.ebook_path==File.path')
|
||||
cover_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE'))
|
||||
cover_file: File = relationship('File', primaryjoin='EBook.cover_path==File.path')
|
||||
|
||||
def __repr__(self):
|
||||
cover = 'yes' if self.cover_file else 'no'
|
||||
return f'<EBook id={self.id} path={repr(self.ebook_path)} cover={cover}>'
|
||||
|
||||
def my_paths(self) -> Generator[pathlib.Path, None, None]:
|
||||
if self.ebook_path:
|
||||
yield self.ebook_path
|
||||
if self.cover_path:
|
||||
yield self.cover_path
|
||||
|
||||
def my_files(self) -> Generator[File, None, None]:
|
||||
if self.ebook_file:
|
||||
yield self.ebook_file
|
||||
if self.cover_file:
|
||||
yield self.cover_file
|
||||
|
||||
def __json__(self):
|
||||
d = dict(
|
||||
cover_path=self.cover_path,
|
||||
creator=self.creator,
|
||||
ebook_path=self.ebook_path,
|
||||
id=self.id,
|
||||
size=self.size,
|
||||
title=self.title,
|
||||
)
|
||||
return d
|
||||
|
||||
def generate_cover(self) -> Optional[pathlib.Path]:
|
||||
"""Discover this ebook's cover, if found, store it next to the ebook file."""
|
||||
path = self.ebook_file.path if self.ebook_file else self.ebook_path
|
||||
|
||||
if not path:
|
||||
raise ValueError('Cannot generate cover for ebook without ebook file!')
|
||||
|
||||
if calibre_cover := discover_calibre_cover(path):
|
||||
return calibre_cover
|
||||
|
||||
cover_bytes = extract_ebook_cover(path)
|
||||
if cover_bytes:
|
||||
cover_path = path.with_suffix('.jpeg')
|
||||
cover_path.write_bytes(cover_bytes)
|
||||
return cover_path
|
||||
|
||||
logger.warning(f'Unable to generate cover for {self.ebook_path}')
|
||||
|
||||
@staticmethod
|
||||
def find_by_path(path, session) -> Base:
|
||||
return session.query(EBook).filter(EBook.ebook_path == path).one_or_none()
|
||||
|
||||
|
||||
def model_ebook(session: Session, ebook_file: File, files: List[File]) -> EBook:
|
||||
"""
|
||||
Creates an EBook model based off a File. Searches for it's cover in the provided `files`.
|
||||
"""
|
||||
# Multiple formats may share this cover.
|
||||
cover_file = next((i for i in files if i.mimetype.split('/')[0] == 'image'), None)
|
||||
|
||||
ebook = session.query(EBook).filter_by(ebook_file=ebook_file).one_or_none()
|
||||
if not ebook:
|
||||
ebook = EBook(ebook_file=ebook_file)
|
||||
session.add(ebook)
|
||||
|
||||
size = ebook_file.path.stat().st_size
|
||||
|
||||
ebook.ebook_file.do_index()
|
||||
|
||||
index = size != ebook.size or cover_file != ebook.cover_file or not ebook.title
|
||||
if not ebook.ebook_file.indexed or index:
|
||||
try:
|
||||
# Only read the contents of the file if it has changed.
|
||||
data = extract_ebook_data(ebook_file.path, ebook_file.mimetype)
|
||||
if data:
|
||||
# Title is a_text.
|
||||
ebook.ebook_file.a_text = ebook.title = data.title
|
||||
# Creator is b_text.
|
||||
ebook.ebook_file.b_text = ebook.creator = data.creator
|
||||
# All text is d_text.
|
||||
ebook.ebook_file.d_text = data.text
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract book data', exc_info=e)
|
||||
|
||||
if not ebook.title:
|
||||
# Book was not indexed above (probably a MOBI), use the file data.
|
||||
name, _ = split_path_stem_and_suffix(ebook.ebook_file.path)
|
||||
ebook.title = ebook.ebook_file.a_text = name
|
||||
|
||||
if cover_file:
|
||||
cover_file.associated = True
|
||||
|
||||
ebook.cover_file = cover_file
|
||||
ebook.size = size
|
||||
ebook_file.model = EBook.__tablename__
|
||||
|
||||
return ebook
|
||||
|
||||
|
||||
@register_modeler
|
||||
def ebook_modeler(groups: Dict[str, List[File]], session: Session):
|
||||
"""Searches for ebook files and models them into the ebook table.
|
||||
|
||||
May generate cover files for EPUBs."""
|
||||
local_groups = groups.copy()
|
||||
|
||||
for stem, group in local_groups.items():
|
||||
found_ebook = False
|
||||
for file in group:
|
||||
if not mimetype_is_ebook(file.mimetype):
|
||||
continue
|
||||
|
||||
session.flush(group)
|
||||
model_ebook(session, file, group)
|
||||
|
||||
found_ebook = True
|
||||
|
||||
if found_ebook:
|
||||
# Claim this group for this ebook.
|
||||
del groups[stem]
|
||||
|
||||
|
||||
@register_after_refresh
|
||||
def discover_ebook_covers():
|
||||
"""Discovers/extracts ebook covers for any Ebook that is missing one."""
|
||||
with get_db_curs() as curs:
|
||||
stmt = f'''
|
||||
SELECT e.id
|
||||
FROM ebook e
|
||||
LEFT JOIN file f on e.ebook_path = f.path
|
||||
WHERE
|
||||
e.cover_path IS NULL
|
||||
AND f.mimetype LIKE '{EPUB_MIMETYPE}%' '''
|
||||
curs.execute(stmt)
|
||||
missing_covers = [i[0] for i in curs.fetchall()]
|
||||
|
||||
if not missing_covers:
|
||||
logger.info('No ebook covers to generate')
|
||||
return
|
||||
|
||||
logger.warning(f'Discovering/generating {len(missing_covers)} ebook covers')
|
||||
|
||||
for id_ in missing_covers:
|
||||
with get_db_session(commit=True) as session:
|
||||
ebook: EBook = session.query(EBook).filter(EBook.id == id_).one()
|
||||
|
||||
cover_path = ebook.generate_cover()
|
||||
if cover_path:
|
||||
# Cover was generated or discovered. Associate it with the ebook.
|
||||
cover_file = session.query(File).filter(File.path == cover_path).one_or_none()
|
||||
if not cover_file:
|
||||
cover_file = File(path=cover_path)
|
||||
session.add(cover_file)
|
||||
cover_file.do_index()
|
||||
cover_file.associated = True
|
||||
|
||||
ebook.cover_file = cover_file
|
||||
|
|
@ -5,6 +5,11 @@ from collections import defaultdict
|
|||
from typing import List, Type, Tuple
|
||||
from zipfile import ZipFile
|
||||
|
||||
try:
|
||||
from PyPDF2 import PdfReader
|
||||
except ImportError:
|
||||
PdfReader = None
|
||||
|
||||
from wrolpi.common import logger
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -45,10 +50,10 @@ def find_indexer(file) -> Type[Indexer]:
|
|||
if not file.mimetype:
|
||||
return DefaultIndexer
|
||||
|
||||
if file.mimetype in indexer_map:
|
||||
# Use the specific indexer.
|
||||
indexer = indexer_map[file.mimetype]
|
||||
else:
|
||||
# Get the indexer that matches the mimetype of this file.
|
||||
indexer = next((v for k, v in indexer_map.items() if file.mimetype.startswith(k)), None)
|
||||
|
||||
if not indexer:
|
||||
# Use the broad indexer.
|
||||
indexer = indexer_map[file.mimetype.split('/')[0]]
|
||||
|
||||
|
|
@ -126,3 +131,36 @@ class TextIndexer(Indexer, ABC):
|
|||
# Some text files aren't for humans.
|
||||
return DefaultIndexer
|
||||
return cls
|
||||
|
||||
|
||||
@register_indexer('application/pdf')
|
||||
class PDFIndexer(Indexer, ABC):
|
||||
"""Uses PyPDF2 to extract text from a PDF."""
|
||||
|
||||
@classmethod
|
||||
def create_index(cls, file):
|
||||
path = file.path.path if hasattr(file.path, 'path') else file.path
|
||||
a = cls.get_title(file)
|
||||
try:
|
||||
# PDFs are complex, don't fail to create title index if text extraction fails.
|
||||
words = cls.get_words(path)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index {path}', exc_info=e)
|
||||
words = ''
|
||||
return a, None, None, words
|
||||
|
||||
@classmethod
|
||||
def get_words(cls, path: pathlib.Path) -> str:
|
||||
if PdfReader is None:
|
||||
return ''
|
||||
|
||||
reader = PdfReader(path)
|
||||
words = []
|
||||
for page in reader.pages:
|
||||
# Get all text facing up.
|
||||
text = page.extract_text(0)
|
||||
words.append(text)
|
||||
words = '\n'.join(words)
|
||||
# Postgres does not allow null characters.
|
||||
words = words.replace('\x00', '\uFFFD')
|
||||
return words
|
||||
|
|
|
|||
|
|
@ -28,11 +28,10 @@ try:
|
|||
import magic
|
||||
|
||||
mime = magic.Magic(mime=True)
|
||||
|
||||
no_magic = False
|
||||
except ImportError:
|
||||
# Magic is not installed
|
||||
no_magic = True
|
||||
magic = None
|
||||
mime = None
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -95,9 +94,20 @@ FILE_NAME_REGEX = re.compile(r'[_ .]')
|
|||
FILE_BIN = which('file', '/usr/bin/file')
|
||||
|
||||
|
||||
def _mimetype_suffix_map(path: Path, mimetype: str):
|
||||
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
|
||||
from wrolpi.files.ebooks import MOBI_MIMETYPE
|
||||
suffix = path.suffix.lower()
|
||||
if mimetype == 'application/octet-stream' and suffix.endswith('.mobi'):
|
||||
return MOBI_MIMETYPE
|
||||
if mimetype == 'text/plain' and suffix.endswith('.json'):
|
||||
return 'application/json'
|
||||
return mimetype
|
||||
|
||||
|
||||
def get_mimetype(path: Path) -> str:
|
||||
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
|
||||
if no_magic:
|
||||
if magic is None:
|
||||
# This method is slow, prefer the speedier `magic` module.
|
||||
cmd = (FILE_BIN, '--mime-type', str(path.absolute()))
|
||||
output = subprocess.check_output(cmd)
|
||||
|
|
@ -105,7 +115,9 @@ def get_mimetype(path: Path) -> str:
|
|||
mimetype = output.split(' ')[-1].strip()
|
||||
return mimetype
|
||||
else:
|
||||
return mime.from_file(path)
|
||||
mimetype = mime.from_file(path)
|
||||
mimetype = _mimetype_suffix_map(path, mimetype)
|
||||
return mimetype
|
||||
|
||||
|
||||
# Special suffixes within WROLPi.
|
||||
|
|
@ -328,17 +340,18 @@ async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str
|
|||
refresh_logger.info(f'Done refreshing files in {directory}')
|
||||
|
||||
|
||||
def search_files(search_str: str, limit: int, offset: int, mimetype: str = None, model: str = None) -> \
|
||||
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None) -> \
|
||||
Tuple[List[dict], int]:
|
||||
"""
|
||||
Search the Files table. Order the returned Files by their rank if `search_str` is provided. Return all files if
|
||||
"""Search the Files table.
|
||||
|
||||
Order the returned Files by their rank if `search_str` is provided. Return all files if
|
||||
`search_str` is empty.
|
||||
|
||||
Parameters:
|
||||
search_str: Search the ts_vector of the file. Returns all files if this is empty.
|
||||
limit: Return only this many files.
|
||||
offset: Offset the query.
|
||||
mimetype: Only return files that match this mimetype.
|
||||
mimetypes: Only return files that match these mimetypes.
|
||||
model: Only return files that match this model.
|
||||
"""
|
||||
params = dict(offset=offset, limit=limit)
|
||||
|
|
@ -352,12 +365,17 @@ def search_files(search_str: str, limit: int, offset: int, mimetype: str = None,
|
|||
selects.append('ts_rank(textsearch, websearch_to_tsquery(%(search_str)s))')
|
||||
order_by = '2 DESC'
|
||||
|
||||
if mimetype and len(mimetype.split('/')) == 1:
|
||||
params['mimetype'] = f'{mimetype}/%'
|
||||
wheres.append('mimetype LIKE %(mimetype)s')
|
||||
elif mimetype:
|
||||
wheres.append('mimetype = %(mimetype)s')
|
||||
params['mimetype'] = mimetype
|
||||
if mimetypes:
|
||||
mimetype_wheres = []
|
||||
for idx, mimetype in enumerate(mimetypes):
|
||||
key = f'mimetype{idx}'
|
||||
if len(mimetype.split('/')) == 1:
|
||||
params[key] = f'{mimetype}/%'
|
||||
else:
|
||||
params[key] = f'{mimetype}%'
|
||||
mimetype_wheres.append(f'mimetype LIKE %({key})s')
|
||||
mimetype_wheres = ' OR '.join(mimetype_wheres)
|
||||
wheres.append(f'({mimetype_wheres})')
|
||||
|
||||
if model:
|
||||
params['model'] = model
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class FilesSearchRequest:
|
|||
search_str: Optional[str] = None
|
||||
limit: Optional[int] = 20
|
||||
offset: Optional[int] = 0
|
||||
mimetype: Optional[str] = None
|
||||
mimetypes: List[str] = field(default_factory=list)
|
||||
model: Optional[str] = None
|
||||
|
||||
|
||||
|
|
|
|||
127
wrolpi/files/test/test_ebooks.py
Normal file
127
wrolpi/files/test/test_ebooks.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
import json
|
||||
import shutil
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.files.ebooks import EBook, MOBI_MIMETYPE, EPUB_MIMETYPE
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index(test_session, test_directory, example_epub, example_mobi):
|
||||
"""Ebooks can be indexed.
|
||||
|
||||
Covers can be discovered."""
|
||||
await refresh_files()
|
||||
|
||||
ebook_epub, ebook_mobi = test_session.query(EBook).order_by(EBook.ebook_path)
|
||||
ebook_epub: EBook
|
||||
ebook_mobi: EBook
|
||||
|
||||
assert ebook_epub.title == 'WROLPi Test Book'
|
||||
assert ebook_epub.ebook_file.mimetype.startswith(EPUB_MIMETYPE)
|
||||
assert ebook_epub.creator == 'roland'
|
||||
assert ebook_epub.size == ebook_epub.ebook_file.size
|
||||
assert ebook_epub.cover_path and ebook_epub.cover_path.is_file()
|
||||
# The cover is the WROLPi logo.
|
||||
assert ebook_epub.cover_file.size == 297099
|
||||
|
||||
assert ebook_epub.ebook_file.a_text, 'Book title was not indexed'
|
||||
assert ebook_epub.ebook_file.b_text, 'Book creator was not indexed'
|
||||
assert ebook_epub.ebook_file.d_text, 'Book text was not indexed'
|
||||
|
||||
# Mobi is not fully supported, title is the file name. No creator or cover.
|
||||
assert ebook_mobi.title == 'example'
|
||||
assert ebook_mobi.ebook_file.mimetype == MOBI_MIMETYPE
|
||||
assert not ebook_mobi.creator
|
||||
assert not ebook_mobi.cover_path
|
||||
|
||||
# Ebooks can be deleted during refresh.
|
||||
example_mobi.unlink()
|
||||
example_epub.unlink()
|
||||
await refresh_files()
|
||||
assert test_session.query(EBook).count() == 0
|
||||
|
||||
|
||||
def test_search(test_session, test_client, example_epub):
|
||||
"""Ebooks are handled in File search results."""
|
||||
request, response = test_client.post('/api/files/refresh')
|
||||
assert response.status == HTTPStatus.NO_CONTENT
|
||||
|
||||
assert test_session.query(EBook).count() == 1
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.creator == 'roland'
|
||||
assert ebook.ebook_file.a_text == 'WROLPi Test Book', 'Book title was not updated'
|
||||
assert ebook.ebook_file.d_text, 'Book was not indexed'
|
||||
|
||||
content = dict(mimetypes=['application/epub', 'application/x-mobipocket-ebook'])
|
||||
request, response = test_client.post('/api/files/search', content=json.dumps(content))
|
||||
assert response.status == HTTPStatus.OK
|
||||
assert response.json
|
||||
result = response.json['files'][0]
|
||||
assert result['path'] == 'example.epub'
|
||||
assert result['mimetype'] == 'application/epub+zip'
|
||||
assert_dict_contains(
|
||||
result['ebook'],
|
||||
{'cover_path': 'example.jpeg', 'ebook_path': 'example.epub'},
|
||||
)
|
||||
|
||||
content = dict(mimetypes=['application/epub+zip', 'application/x-mobipocket-ebook'])
|
||||
request, response = test_client.post('/api/files/search', content=json.dumps(content))
|
||||
result = response.json['files'][0]
|
||||
assert result['path'] == 'example.epub'
|
||||
assert result['mimetype'] == 'application/epub+zip'
|
||||
|
||||
content = dict(mimetypes=[])
|
||||
request, response = test_client.post('/api/files/search', content=json.dumps(content))
|
||||
assert response.status == HTTPStatus.OK
|
||||
assert response.json
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_calibre_cover(test_session, test_directory, example_epub, image_file):
|
||||
"""Calibre puts a cover near an ebook file, test if it can be found."""
|
||||
# Create two files in the
|
||||
metadata = test_directory / 'metadata.opf'
|
||||
metadata.touch()
|
||||
# Create an alternate format next to the epub. This will be ignored because it is not yet supported.
|
||||
mobi_path = example_epub.with_suffix('.mobi')
|
||||
mobi_path.touch()
|
||||
|
||||
await refresh_files()
|
||||
|
||||
assert test_session.query(EBook).count() == 1
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
# The cover in the ebook was extracted.
|
||||
assert ebook.cover_file.size == 297099
|
||||
|
||||
# Delete the extracted cover, use the cover image from Calibre.
|
||||
ebook.cover_path.unlink()
|
||||
test_session.delete(ebook.cover_file)
|
||||
cover_image = test_directory / 'cover.jpg'
|
||||
shutil.move(image_file, cover_image)
|
||||
|
||||
# Calibre cover image was discovered, no cover was generated.
|
||||
await refresh_files()
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.cover_file.size == 641
|
||||
|
||||
# Create a file nearby, its no longer a Calibre directory.
|
||||
mobi_path = mobi_path.rename(test_directory / 'other book.mobi')
|
||||
await refresh_files()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.cover_file.size == 297099
|
||||
mobi_path.unlink()
|
||||
|
||||
# Delete the discovered cover, cover should be generated.
|
||||
cover_image.unlink()
|
||||
await refresh_files()
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.title == 'WROLPi Test Book'
|
||||
assert ebook.cover_file.size == 297099
|
||||
|
|
@ -469,3 +469,25 @@ def test_matching_directories(make_files_structure, test_directory):
|
|||
# foo is an exact match, return subdirectories
|
||||
matches = lib.get_matching_directories(test_directory / 'foo')
|
||||
assert matches == [str(test_directory / 'foo/qux')]
|
||||
|
||||
|
||||
def test_pdf_indexer(example_pdf):
|
||||
"""PDFs can be indexed by PDFIndexer."""
|
||||
file = File(path=example_pdf)
|
||||
|
||||
a_text, b_text, c_text, d_text = indexers.PDFIndexer.create_index(file)
|
||||
|
||||
# The file name.
|
||||
assert a_text == ('example', '.pdf')
|
||||
assert b_text is None
|
||||
assert c_text is None
|
||||
# Both pages of the example PDF are extracted.
|
||||
assert d_text == 'Page one\nPage two'
|
||||
|
||||
|
||||
def test_get_mimetype(example_epub, example_mobi, example_pdf, image_file, video_file):
|
||||
assert lib.get_mimetype(example_epub) == 'application/epub+zip'
|
||||
assert lib.get_mimetype(example_mobi) == 'application/x-mobipocket-ebook'
|
||||
assert lib.get_mimetype(example_pdf) == 'application/pdf'
|
||||
assert lib.get_mimetype(image_file) == 'image/jpeg'
|
||||
assert lib.get_mimetype(video_file) == 'video/mp4'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue