mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Add Docs module for searching and organizing PDFs and EPUBs
Create a new Docs module that consolidates PDF and eBook handling into a unified system with subjects and authors support. This replaces the separate PDF/eBook modelers with a single Doc model backed by an alembic migration (rename ebook to doc). Key changes: - New modules/docs package with models, API, extractors, and lib - Subject and author metadata extraction with case-insensitive normalization for cleaner PDF metadata - Frontend Docs pages with AuthorsPage, SubjectsPage using CollectionTable, and doc card deletion - Global search integration for subjects and authors - Generic collection item count support - Frontend file card metadata adjusted to match collection sort - Bump yt-dlp version
This commit is contained in:
parent
4a21499bdd
commit
aefc7b1def
90 changed files with 4484 additions and 1189 deletions
|
|
@ -18,7 +18,7 @@ fileConfig(config.config_file_name)
|
|||
from wrolpi.common import Base
|
||||
# Import all models so they are registered with the Base metadata
|
||||
from wrolpi.files.models import FileGroup, Directory # noqa: F401
|
||||
from wrolpi.files.ebooks import EBook # noqa: F401
|
||||
from modules.docs.models import Doc # noqa: F401
|
||||
from wrolpi.tags import Tag # noqa: F401
|
||||
from wrolpi.flags import WROLPiFlag # noqa: F401
|
||||
from wrolpi.downloader import Download # noqa: F401
|
||||
|
|
|
|||
172
alembic/versions/e7f8a9b0c1d2_rename_ebook_to_doc.py
Normal file
172
alembic/versions/e7f8a9b0c1d2_rename_ebook_to_doc.py
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
"""Rename ebook table to doc and add new columns.
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises: d371333159ab
|
||||
Create Date: 2026-03-15 15:30:00.000000
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'e7f8a9b0c1d2'
|
||||
down_revision = 'd371333159ab'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
if not DOCKERIZED:
|
||||
bind.execute(sa.text("SET ROLE 'wrolpi'"))
|
||||
|
||||
# Rename ebook table to doc.
|
||||
print("Migrating ebook -> doc...")
|
||||
op.rename_table('ebook', 'doc')
|
||||
|
||||
# Rename the index.
|
||||
op.execute('ALTER INDEX IF EXISTS ebook_size_idx RENAME TO doc_size_idx')
|
||||
op.execute('ALTER INDEX IF EXISTS ebook_file_group_id_key RENAME TO doc_file_group_id_key')
|
||||
op.execute('ALTER INDEX IF EXISTS ebook_pkey RENAME TO doc_pkey')
|
||||
|
||||
# Rename the sequence.
|
||||
op.execute('ALTER SEQUENCE IF EXISTS ebook_id_seq RENAME TO doc_id_seq')
|
||||
|
||||
# Add new columns.
|
||||
op.add_column('doc', sa.Column('publisher', sa.String(), nullable=True))
|
||||
op.add_column('doc', sa.Column('language', sa.String(), nullable=True))
|
||||
op.add_column('doc', sa.Column('page_count', sa.Integer(), nullable=True))
|
||||
op.add_column('doc', sa.Column('subject', sa.String(), nullable=True))
|
||||
op.add_column('doc', sa.Column('description', sa.Text(), nullable=True))
|
||||
|
||||
# Widen size from Integer to BigInteger.
|
||||
op.alter_column('doc', 'size', type_=sa.BigInteger(), existing_type=sa.Integer())
|
||||
|
||||
# Update file_group.model: 'ebook' -> 'doc'.
|
||||
# Mark all touched file_groups as not indexed so metadata is re-extracted on next refresh.
|
||||
total = 0
|
||||
while True:
|
||||
result = bind.execute(sa.text(
|
||||
"UPDATE file_group SET model = 'doc', indexed = false"
|
||||
" WHERE id IN (SELECT id FROM file_group WHERE model = 'ebook' LIMIT :batch_size)"
|
||||
), {'batch_size': BATCH_SIZE})
|
||||
if result.rowcount == 0:
|
||||
break
|
||||
total += result.rowcount
|
||||
print(f" Updated {total} ebook file_groups -> doc...")
|
||||
session.commit()
|
||||
print(f" Updated {total} ebook file_groups -> doc (done)")
|
||||
|
||||
# Update file_group.model: 'pdf' -> 'doc'.
|
||||
total = 0
|
||||
while True:
|
||||
result = bind.execute(sa.text(
|
||||
"UPDATE file_group SET model = 'doc', indexed = false"
|
||||
" WHERE id IN (SELECT id FROM file_group WHERE model = 'pdf' LIMIT :batch_size)"
|
||||
), {'batch_size': BATCH_SIZE})
|
||||
if result.rowcount == 0:
|
||||
break
|
||||
total += result.rowcount
|
||||
print(f" Updated {total} pdf file_groups -> doc...")
|
||||
session.commit()
|
||||
print(f" Updated {total} pdf file_groups -> doc (done)")
|
||||
|
||||
# Create Doc records for existing PDFs that had no model table.
|
||||
# Metadata will be extracted when the doc_modeler re-indexes these files.
|
||||
total = 0
|
||||
while True:
|
||||
result = bind.execute(sa.text("""
|
||||
INSERT INTO doc (file_group_id)
|
||||
SELECT fg.id
|
||||
FROM file_group fg
|
||||
LEFT JOIN doc d ON d.file_group_id = fg.id
|
||||
WHERE fg.mimetype = 'application/pdf'
|
||||
AND d.id IS NULL
|
||||
AND fg.primary_path IS NOT NULL
|
||||
LIMIT :batch_size
|
||||
"""), {'batch_size': BATCH_SIZE})
|
||||
if result.rowcount == 0:
|
||||
break
|
||||
total += result.rowcount
|
||||
print(f" Created {total} new doc records for existing PDFs...")
|
||||
session.commit()
|
||||
print(f" Created {total} new doc records for existing PDFs (done)")
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
if not DOCKERIZED:
|
||||
bind.execute(sa.text("SET ROLE 'wrolpi'"))
|
||||
|
||||
# Delete doc records that were PDFs (have no ebook equivalent).
|
||||
total = 0
|
||||
while True:
|
||||
result = bind.execute(sa.text("""
|
||||
DELETE FROM doc
|
||||
WHERE id IN (
|
||||
SELECT d.id FROM doc d
|
||||
JOIN file_group fg ON fg.id = d.file_group_id
|
||||
WHERE fg.mimetype = 'application/pdf'
|
||||
LIMIT :batch_size
|
||||
)
|
||||
"""), {'batch_size': BATCH_SIZE})
|
||||
if result.rowcount == 0:
|
||||
break
|
||||
total += result.rowcount
|
||||
print(f" Deleted {total} PDF doc records...")
|
||||
session.commit()
|
||||
print(f" Deleted {total} PDF doc records (done)")
|
||||
|
||||
# Revert file_group.model and re-mark as not indexed.
|
||||
for old_model, mimetype_clause in [
|
||||
('ebook', "mimetype LIKE 'application/epub%'"),
|
||||
('ebook', "mimetype = 'application/x-mobipocket-ebook'"),
|
||||
('pdf', "mimetype = 'application/pdf'"),
|
||||
]:
|
||||
total = 0
|
||||
while True:
|
||||
result = bind.execute(sa.text(f"""
|
||||
UPDATE file_group SET model = :old_model, indexed = false
|
||||
WHERE id IN (
|
||||
SELECT id FROM file_group
|
||||
WHERE model = 'doc' AND {mimetype_clause}
|
||||
LIMIT :batch_size
|
||||
)
|
||||
"""), {'old_model': old_model, 'batch_size': BATCH_SIZE})
|
||||
if result.rowcount == 0:
|
||||
break
|
||||
total += result.rowcount
|
||||
print(f" Reverted {total} file_groups -> {old_model} ({mimetype_clause})...")
|
||||
session.commit()
|
||||
print(f" Reverted {total} file_groups -> {old_model} ({mimetype_clause}) (done)")
|
||||
|
||||
# Revert size column type.
|
||||
op.alter_column('doc', 'size', type_=sa.Integer(), existing_type=sa.BigInteger())
|
||||
|
||||
# Drop new columns.
|
||||
op.drop_column('doc', 'description')
|
||||
op.drop_column('doc', 'subject')
|
||||
op.drop_column('doc', 'page_count')
|
||||
op.drop_column('doc', 'language')
|
||||
op.drop_column('doc', 'publisher')
|
||||
|
||||
# Rename sequences back.
|
||||
op.execute('ALTER SEQUENCE IF EXISTS doc_id_seq RENAME TO ebook_id_seq')
|
||||
|
||||
# Rename indexes back.
|
||||
op.execute('ALTER INDEX IF EXISTS doc_pkey RENAME TO ebook_pkey')
|
||||
op.execute('ALTER INDEX IF EXISTS doc_file_group_id_key RENAME TO ebook_file_group_id_key')
|
||||
op.execute('ALTER INDEX IF EXISTS doc_size_idx RENAME TO ebook_size_idx')
|
||||
|
||||
# Rename table back.
|
||||
op.rename_table('doc', 'ebook')
|
||||
86
app/package-lock.json
generated
86
app/package-lock.json
generated
|
|
@ -97,6 +97,7 @@
|
|||
"version": "7.20.12",
|
||||
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz",
|
||||
"integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@ampproject/remapping": "^2.1.0",
|
||||
"@babel/code-frame": "^7.18.6",
|
||||
|
|
@ -903,6 +904,7 @@
|
|||
"version": "7.18.6",
|
||||
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz",
|
||||
"integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/helper-plugin-utils": "^7.18.6"
|
||||
},
|
||||
|
|
@ -1464,6 +1466,7 @@
|
|||
"version": "7.20.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.20.13.tgz",
|
||||
"integrity": "sha512-MmTZx/bkUrfJhhYAYt3Urjm+h8DQGrPrnKQ94jLo7NLuOU+T89a7IByhKmrb8SKhrIYIQ0FN0CHMbnFRen4qNw==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/helper-annotate-as-pure": "^7.18.6",
|
||||
"@babel/helper-module-imports": "^7.18.6",
|
||||
|
|
@ -3283,6 +3286,7 @@
|
|||
"version": "2.11.6",
|
||||
"resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz",
|
||||
"integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==",
|
||||
"peer": true,
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/popperjs"
|
||||
|
|
@ -4066,6 +4070,7 @@
|
|||
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.27.tgz",
|
||||
"integrity": "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@types/prop-types": "*",
|
||||
"csstype": "^3.2.2"
|
||||
|
|
@ -4076,6 +4081,7 @@
|
|||
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz",
|
||||
"integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"peerDependencies": {
|
||||
"@types/react": "^18.0.0"
|
||||
}
|
||||
|
|
@ -4190,6 +4196,7 @@
|
|||
"version": "5.50.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.50.0.tgz",
|
||||
"integrity": "sha512-vwksQWSFZiUhgq3Kv7o1Jcj0DUNylwnIlGvKvLLYsq8pAWha6/WCnXUeaSoNNha/K7QSf2+jvmkxggC1u3pIwQ==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/scope-manager": "5.50.0",
|
||||
"@typescript-eslint/type-utils": "5.50.0",
|
||||
|
|
@ -4241,6 +4248,7 @@
|
|||
"version": "5.50.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.50.0.tgz",
|
||||
"integrity": "sha512-KCcSyNaogUDftK2G9RXfQyOCt51uB5yqC6pkUYqhYh8Kgt+DwR5M0EwEAxGPy/+DH6hnmKeGsNhiZRQxjH71uQ==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/scope-manager": "5.50.0",
|
||||
"@typescript-eslint/types": "5.50.0",
|
||||
|
|
@ -4571,6 +4579,7 @@
|
|||
"version": "8.15.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
|
||||
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"acorn": "bin/acorn"
|
||||
},
|
||||
|
|
@ -4693,6 +4702,7 @@
|
|||
"version": "6.14.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
|
||||
"integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.1",
|
||||
"fast-json-stable-stringify": "^2.0.0",
|
||||
|
|
@ -5550,6 +5560,7 @@
|
|||
"url": "https://github.com/sponsors/ai"
|
||||
}
|
||||
],
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"baseline-browser-mapping": "^2.9.0",
|
||||
"caniuse-lite": "^1.0.30001759",
|
||||
|
|
@ -6347,6 +6358,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -6633,6 +6645,7 @@
|
|||
"resolved": "https://registry.npmjs.org/cypress/-/cypress-13.15.2.tgz",
|
||||
"integrity": "sha512-ARbnUorjcCM3XiPwgHKuqsyr5W9Qn+pIIBPaoilnoBkLdSC2oLQjV1BUpnmc7KR+b7Avah3Ly2RMFnfxr96E/A==",
|
||||
"hasInstallScript": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@cypress/request": "^3.0.6",
|
||||
"@cypress/xvfb": "^1.2.4",
|
||||
|
|
@ -7393,6 +7406,7 @@
|
|||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz",
|
||||
"integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"ansi-colors": "^4.1.1",
|
||||
"strip-ansi": "^6.0.1"
|
||||
|
|
@ -7672,6 +7686,7 @@
|
|||
"version": "8.33.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.33.0.tgz",
|
||||
"integrity": "sha512-WjOpFQgKK8VrCnAtl8We0SUOy/oVZ5NHykyMiagV1M9r8IFpIJX7DduK6n1mpfhlG7T1NLWm2SuD8QB7KFySaA==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@eslint/eslintrc": "^1.4.1",
|
||||
"@humanwhocodes/config-array": "^0.11.8",
|
||||
|
|
@ -8085,6 +8100,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -10570,6 +10586,7 @@
|
|||
"version": "27.5.1",
|
||||
"resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz",
|
||||
"integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@jest/core": "^27.5.1",
|
||||
"import-local": "^3.0.2",
|
||||
|
|
@ -13214,6 +13231,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -14022,6 +14040,7 @@
|
|||
"url": "https://tidelift.com/funding/github/npm/postcss"
|
||||
}
|
||||
],
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"nanoid": "^3.3.4",
|
||||
"picocolors": "^1.0.0",
|
||||
|
|
@ -15080,6 +15099,7 @@
|
|||
"version": "6.0.11",
|
||||
"resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz",
|
||||
"integrity": "sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"cssesc": "^3.0.0",
|
||||
"util-deprecate": "^1.0.2"
|
||||
|
|
@ -15269,6 +15289,7 @@
|
|||
"version": "15.8.1",
|
||||
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
|
||||
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"loose-envify": "^1.4.0",
|
||||
"object-assign": "^4.1.1",
|
||||
|
|
@ -15459,6 +15480,7 @@
|
|||
"version": "18.2.0",
|
||||
"resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
|
||||
"integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"loose-envify": "^1.1.0"
|
||||
},
|
||||
|
|
@ -15612,6 +15634,7 @@
|
|||
"version": "18.2.0",
|
||||
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz",
|
||||
"integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"loose-envify": "^1.1.0",
|
||||
"scheduler": "^0.23.0"
|
||||
|
|
@ -15723,6 +15746,7 @@
|
|||
"version": "0.11.0",
|
||||
"resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz",
|
||||
"integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
|
|
@ -16228,6 +16252,7 @@
|
|||
"version": "2.80.0",
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-2.80.0.tgz",
|
||||
"integrity": "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ==",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"rollup": "dist/bin/rollup"
|
||||
},
|
||||
|
|
@ -16503,6 +16528,7 @@
|
|||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.4.tgz",
|
||||
"integrity": "sha512-7CxjBoFUfH7fUvtn+SPkkIocthUD9kV3niF1mUMa9TbeyPAf2brtRCZBlT2OpHaXmkscFzGjEfhbJo9gKfotzQ==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.10.5",
|
||||
"@fluentui/react-component-event-listener": "~0.63.0",
|
||||
|
|
@ -17581,6 +17607,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -17652,7 +17679,8 @@
|
|||
"node_modules/three": {
|
||||
"version": "0.154.0",
|
||||
"resolved": "https://registry.npmjs.org/three/-/three-0.154.0.tgz",
|
||||
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug=="
|
||||
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug==",
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/three-stdlib": {
|
||||
"version": "2.17.2",
|
||||
|
|
@ -17895,6 +17923,7 @@
|
|||
"version": "0.21.3",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz",
|
||||
"integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
|
|
@ -17948,6 +17977,7 @@
|
|||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
|
||||
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
|
|
@ -18259,6 +18289,7 @@
|
|||
"version": "5.105.0",
|
||||
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz",
|
||||
"integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@types/eslint-scope": "^3.7.7",
|
||||
"@types/estree": "^1.0.8",
|
||||
|
|
@ -18328,6 +18359,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -18377,6 +18409,7 @@
|
|||
"version": "4.11.1",
|
||||
"resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz",
|
||||
"integrity": "sha512-lILVz9tAUy1zGFwieuaQtYiadImb5M3d+H+L1zDYalYoDl0cksAB1UNyuE5MMWJrG6zR1tXkCP2fitl7yoUJiw==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@types/bonjour": "^3.5.9",
|
||||
"@types/connect-history-api-fallback": "^1.3.5",
|
||||
|
|
@ -18431,6 +18464,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -18543,6 +18577,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -18825,6 +18860,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -19264,6 +19300,7 @@
|
|||
"version": "7.20.12",
|
||||
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz",
|
||||
"integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@ampproject/remapping": "^2.1.0",
|
||||
"@babel/code-frame": "^7.18.6",
|
||||
|
|
@ -19813,6 +19850,7 @@
|
|||
"version": "7.18.6",
|
||||
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz",
|
||||
"integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@babel/helper-plugin-utils": "^7.18.6"
|
||||
}
|
||||
|
|
@ -20158,6 +20196,7 @@
|
|||
"version": "7.20.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.20.13.tgz",
|
||||
"integrity": "sha512-MmTZx/bkUrfJhhYAYt3Urjm+h8DQGrPrnKQ94jLo7NLuOU+T89a7IByhKmrb8SKhrIYIQ0FN0CHMbnFRen4qNw==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@babel/helper-annotate-as-pure": "^7.18.6",
|
||||
"@babel/helper-module-imports": "^7.18.6",
|
||||
|
|
@ -21423,7 +21462,8 @@
|
|||
"@popperjs/core": {
|
||||
"version": "2.11.6",
|
||||
"resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz",
|
||||
"integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw=="
|
||||
"integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==",
|
||||
"peer": true
|
||||
},
|
||||
"@react-three/fiber": {
|
||||
"version": "8.16.5",
|
||||
|
|
@ -21996,6 +22036,7 @@
|
|||
"version": "18.3.27",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.27.tgz",
|
||||
"integrity": "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@types/prop-types": "*",
|
||||
"csstype": "^3.2.2"
|
||||
|
|
@ -22005,6 +22046,7 @@
|
|||
"version": "18.3.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz",
|
||||
"integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==",
|
||||
"peer": true,
|
||||
"requires": {}
|
||||
},
|
||||
"@types/react-reconciler": {
|
||||
|
|
@ -22117,6 +22159,7 @@
|
|||
"version": "5.50.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.50.0.tgz",
|
||||
"integrity": "sha512-vwksQWSFZiUhgq3Kv7o1Jcj0DUNylwnIlGvKvLLYsq8pAWha6/WCnXUeaSoNNha/K7QSf2+jvmkxggC1u3pIwQ==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@typescript-eslint/scope-manager": "5.50.0",
|
||||
"@typescript-eslint/type-utils": "5.50.0",
|
||||
|
|
@ -22142,6 +22185,7 @@
|
|||
"version": "5.50.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.50.0.tgz",
|
||||
"integrity": "sha512-KCcSyNaogUDftK2G9RXfQyOCt51uB5yqC6pkUYqhYh8Kgt+DwR5M0EwEAxGPy/+DH6hnmKeGsNhiZRQxjH71uQ==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@typescript-eslint/scope-manager": "5.50.0",
|
||||
"@typescript-eslint/types": "5.50.0",
|
||||
|
|
@ -22391,7 +22435,8 @@
|
|||
"acorn": {
|
||||
"version": "8.15.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
|
||||
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="
|
||||
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
|
||||
"peer": true
|
||||
},
|
||||
"acorn-globals": {
|
||||
"version": "6.0.0",
|
||||
|
|
@ -22478,6 +22523,7 @@
|
|||
"version": "6.14.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
|
||||
"integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.1",
|
||||
"fast-json-stable-stringify": "^2.0.0",
|
||||
|
|
@ -23095,6 +23141,7 @@
|
|||
"version": "4.28.1",
|
||||
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz",
|
||||
"integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"baseline-browser-mapping": "^2.9.0",
|
||||
"caniuse-lite": "^1.0.30001759",
|
||||
|
|
@ -23636,6 +23683,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -23844,6 +23892,7 @@
|
|||
"version": "13.15.2",
|
||||
"resolved": "https://registry.npmjs.org/cypress/-/cypress-13.15.2.tgz",
|
||||
"integrity": "sha512-ARbnUorjcCM3XiPwgHKuqsyr5W9Qn+pIIBPaoilnoBkLdSC2oLQjV1BUpnmc7KR+b7Avah3Ly2RMFnfxr96E/A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@cypress/request": "^3.0.6",
|
||||
"@cypress/xvfb": "^1.2.4",
|
||||
|
|
@ -24402,6 +24451,7 @@
|
|||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz",
|
||||
"integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"ansi-colors": "^4.1.1",
|
||||
"strip-ansi": "^6.0.1"
|
||||
|
|
@ -24620,6 +24670,7 @@
|
|||
"version": "8.33.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.33.0.tgz",
|
||||
"integrity": "sha512-WjOpFQgKK8VrCnAtl8We0SUOy/oVZ5NHykyMiagV1M9r8IFpIJX7DduK6n1mpfhlG7T1NLWm2SuD8QB7KFySaA==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@eslint/eslintrc": "^1.4.1",
|
||||
"@humanwhocodes/config-array": "^0.11.8",
|
||||
|
|
@ -24995,6 +25046,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -26660,6 +26712,7 @@
|
|||
"version": "27.5.1",
|
||||
"resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz",
|
||||
"integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@jest/core": "^27.5.1",
|
||||
"import-local": "^3.0.2",
|
||||
|
|
@ -28588,6 +28641,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -29152,6 +29206,7 @@
|
|||
"version": "8.4.21",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz",
|
||||
"integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"nanoid": "^3.3.4",
|
||||
"picocolors": "^1.0.0",
|
||||
|
|
@ -29735,6 +29790,7 @@
|
|||
"version": "6.0.11",
|
||||
"resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz",
|
||||
"integrity": "sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"cssesc": "^3.0.0",
|
||||
"util-deprecate": "^1.0.2"
|
||||
|
|
@ -29874,6 +29930,7 @@
|
|||
"version": "15.8.1",
|
||||
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
|
||||
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"loose-envify": "^1.4.0",
|
||||
"object-assign": "^4.1.1",
|
||||
|
|
@ -30016,6 +30073,7 @@
|
|||
"version": "18.2.0",
|
||||
"resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
|
||||
"integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"loose-envify": "^1.1.0"
|
||||
}
|
||||
|
|
@ -30129,6 +30187,7 @@
|
|||
"version": "18.2.0",
|
||||
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz",
|
||||
"integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"loose-envify": "^1.1.0",
|
||||
"scheduler": "^0.23.0"
|
||||
|
|
@ -30205,7 +30264,8 @@
|
|||
"react-refresh": {
|
||||
"version": "0.11.0",
|
||||
"resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz",
|
||||
"integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A=="
|
||||
"integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==",
|
||||
"peer": true
|
||||
},
|
||||
"react-router": {
|
||||
"version": "7.12.0",
|
||||
|
|
@ -30566,6 +30626,7 @@
|
|||
"version": "2.80.0",
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-2.80.0.tgz",
|
||||
"integrity": "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fsevents": "~2.3.2"
|
||||
}
|
||||
|
|
@ -30751,6 +30812,7 @@
|
|||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.4.tgz",
|
||||
"integrity": "sha512-7CxjBoFUfH7fUvtn+SPkkIocthUD9kV3niF1mUMa9TbeyPAf2brtRCZBlT2OpHaXmkscFzGjEfhbJo9gKfotzQ==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@babel/runtime": "^7.10.5",
|
||||
"@fluentui/react-component-event-listener": "~0.63.0",
|
||||
|
|
@ -31572,6 +31634,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -31623,7 +31686,8 @@
|
|||
"three": {
|
||||
"version": "0.154.0",
|
||||
"resolved": "https://registry.npmjs.org/three/-/three-0.154.0.tgz",
|
||||
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug=="
|
||||
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug==",
|
||||
"peer": true
|
||||
},
|
||||
"three-stdlib": {
|
||||
"version": "2.17.2",
|
||||
|
|
@ -31820,7 +31884,8 @@
|
|||
"type-fest": {
|
||||
"version": "0.21.3",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz",
|
||||
"integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w=="
|
||||
"integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==",
|
||||
"peer": true
|
||||
},
|
||||
"type-is": {
|
||||
"version": "1.6.18",
|
||||
|
|
@ -31857,7 +31922,8 @@
|
|||
"typescript": {
|
||||
"version": "4.9.5",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
|
||||
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g=="
|
||||
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
|
||||
"peer": true
|
||||
},
|
||||
"unbox-primitive": {
|
||||
"version": "1.0.2",
|
||||
|
|
@ -32084,6 +32150,7 @@
|
|||
"version": "5.105.0",
|
||||
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz",
|
||||
"integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@types/eslint-scope": "^3.7.7",
|
||||
"@types/estree": "^1.0.8",
|
||||
|
|
@ -32116,6 +32183,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -32179,6 +32247,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -32216,6 +32285,7 @@
|
|||
"version": "4.11.1",
|
||||
"resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz",
|
||||
"integrity": "sha512-lILVz9tAUy1zGFwieuaQtYiadImb5M3d+H+L1zDYalYoDl0cksAB1UNyuE5MMWJrG6zR1tXkCP2fitl7yoUJiw==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"@types/bonjour": "^3.5.9",
|
||||
"@types/connect-history-api-fallback": "^1.3.5",
|
||||
|
|
@ -32252,6 +32322,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
@ -32498,6 +32569,7 @@
|
|||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
|
||||
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
|
||||
"peer": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"fast-uri": "^3.0.1",
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import {SemanticToastContainer} from "react-semantic-toasts-2";
|
|||
import {FilePreviewProvider} from "./components/FilePreview";
|
||||
import {TagsProvider} from "./Tags";
|
||||
import {ZimRoute} from "./components/Zim";
|
||||
import {DocsRoute} from "./components/Docs";
|
||||
import ErrorBoundary from "./components/ErrorBoundary";
|
||||
import {KeyboardShortcutsProvider} from "./components/KeyboardShortcutsProvider";
|
||||
|
||||
|
|
@ -90,6 +91,7 @@ const router = createBrowserRouter(createRoutesFromElements(<Route
|
|||
<Route path="more/*" element={<ErrorBoundary><MoreRoute/></ErrorBoundary>}/>
|
||||
<Route path="inventory/*" element={<ErrorBoundary><InventoryRoute/></ErrorBoundary>}/>
|
||||
<Route path='archive/*' element={<ErrorBoundary><ArchiveRoute/></ErrorBoundary>}/>
|
||||
<Route path='docs/*' element={<ErrorBoundary><DocsRoute/></ErrorBoundary>}/>
|
||||
<Route path='map/*' element={<ErrorBoundary><MapRoute/></ErrorBoundary>}/>
|
||||
<Route path='zim/*' element={<ErrorBoundary><ZimRoute/></ErrorBoundary>}/>
|
||||
<Route path='files/*' element={<ErrorBoundary><FilesRoute/></ErrorBoundary>}/>
|
||||
|
|
|
|||
|
|
@ -811,6 +811,38 @@ export async function fetchDomains() {
|
|||
}
|
||||
}
|
||||
|
||||
export async function fetchAuthors() {
|
||||
const response = await apiGet(`${COLLECTIONS_API}?kind=author`);
|
||||
if (response.ok) {
|
||||
let data = await response.json();
|
||||
return [data['collections'], data['totals']['collections']];
|
||||
} else {
|
||||
const message = await getErrorMessage(response, 'Unable to fetch Authors. See server logs.');
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Authors Error',
|
||||
description: message,
|
||||
time: 5000,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchSubjects() {
|
||||
const response = await apiGet(`${COLLECTIONS_API}?kind=subject`);
|
||||
if (response.ok) {
|
||||
let data = await response.json();
|
||||
return [data['collections'], data['totals']['collections']];
|
||||
} else {
|
||||
const message = await getErrorMessage(response, 'Unable to fetch Subjects. See server logs.');
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Subjects Error',
|
||||
description: message,
|
||||
time: 5000,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchChannels() {
|
||||
const response = await apiGet(`${COLLECTIONS_API}?kind=channel`);
|
||||
if (response.ok) {
|
||||
|
|
@ -1378,6 +1410,11 @@ export function downloadArchiveMember(path, member) {
|
|||
window.open(`${API_URI}/files/zip/download?${params}`, '_blank');
|
||||
}
|
||||
|
||||
export function getArchiveMemberUrl(path, member) {
|
||||
const params = new URLSearchParams({path, member});
|
||||
return `${API_URI}/files/zip/download?${params}`;
|
||||
}
|
||||
|
||||
export async function fetchFilesProgress() {
|
||||
const response = await apiGet(`${API_URI}/files/worker_status`);
|
||||
if (response.ok) {
|
||||
|
|
@ -1817,6 +1854,8 @@ export async function searchSuggestions(search_str) {
|
|||
return {
|
||||
channels: content.channels,
|
||||
domains: content.domains,
|
||||
authors: content.authors || [],
|
||||
subjects: content.subjects || [],
|
||||
}
|
||||
} else {
|
||||
console.error('Failed to get file search suggestions!');
|
||||
|
|
@ -2056,3 +2095,60 @@ export async function fetchSuggestedUserAgent() {
|
|||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
// Docs API
|
||||
const DOCS_API = '/api/docs';
|
||||
|
||||
export async function getDocStatistics() {
|
||||
return await apiGet(`${DOCS_API}/statistics`);
|
||||
}
|
||||
|
||||
export async function getDoc(docId) {
|
||||
return await apiGet(`${DOCS_API}/${docId}`);
|
||||
}
|
||||
|
||||
export async function getDocByFileGroup(fileGroupId) {
|
||||
return await apiGet(`${DOCS_API}/view/${fileGroupId}`);
|
||||
}
|
||||
|
||||
export async function deleteDocs(docIds) {
|
||||
const ids = Array.isArray(docIds) ? docIds.join(',') : docIds;
|
||||
return await apiDelete(`${DOCS_API}/${ids}`);
|
||||
}
|
||||
|
||||
export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject) {
|
||||
offset = parseInt(offset || 0);
|
||||
limit = parseInt(limit || DEFAULT_LIMIT);
|
||||
let body = {offset, limit};
|
||||
if (searchStr) {
|
||||
body['search_str'] = searchStr;
|
||||
}
|
||||
if (order) {
|
||||
body['order_by'] = order;
|
||||
}
|
||||
if (tagNames && tagNames.length > 0) {
|
||||
body['tag_names'] = tagNames;
|
||||
}
|
||||
if (author) {
|
||||
body['author'] = author;
|
||||
}
|
||||
if (subject) {
|
||||
body['subject'] = subject;
|
||||
}
|
||||
|
||||
console.debug('searching docs', body);
|
||||
const response = await apiPost(`${DOCS_API}/search`, body);
|
||||
if (response.ok) {
|
||||
let data = await response.json();
|
||||
return [data['file_groups'], data['totals']['file_groups']];
|
||||
} else {
|
||||
const message = await getErrorMessage(response, 'Cannot search docs. See server logs.');
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Unable to search docs',
|
||||
description: message,
|
||||
time: 5000,
|
||||
});
|
||||
throw Error(message);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -346,6 +346,8 @@ function ArchivePage() {
|
|||
|
||||
export function ArchiveCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
const {sort} = useSearchOrder();
|
||||
const sortField = sort ? sort.replace(/^-/, '') : null;
|
||||
const {data, directory} = file;
|
||||
|
||||
// Resolve data paths (filename-only) to full relative paths using directory
|
||||
|
|
@ -386,7 +388,16 @@ export function ArchiveCard({file}) {
|
|||
<span {...s}>{domain}</span>
|
||||
</CardLink>}
|
||||
<CardMeta {...s}>
|
||||
{isoDatetimeToAgoPopup(dt, false)}
|
||||
{sortField === 'published_modified_datetime'
|
||||
? isoDatetimeToAgoPopup(file.published_modified_datetime, false)
|
||||
: sortField === 'download_datetime'
|
||||
? isoDatetimeToAgoPopup(file.download_datetime, false)
|
||||
: sortField === 'size'
|
||||
? humanFileSize(file.size)
|
||||
: sortField === 'viewed'
|
||||
? isoDatetimeToAgoPopup(file.viewed, false)
|
||||
: isoDatetimeToAgoPopup(dt, false)
|
||||
}
|
||||
</CardMeta>
|
||||
<CardDescription>
|
||||
<Link to={`/archive/${data.id}`}>
|
||||
|
|
|
|||
152
app/src/components/CbzViewer.js
Normal file
152
app/src/components/CbzViewer.js
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import React, {useCallback, useEffect, useRef, useState} from 'react';
|
||||
import {getArchiveContents, getArchiveMemberUrl} from '../api';
|
||||
import {Icon, Label, Loader, Message} from 'semantic-ui-react';
|
||||
import {Button} from './Theme';
|
||||
import {useLocalStorage} from './Common';
|
||||
|
||||
const IMAGE_EXTENSIONS = /\.(jpe?g|png|gif|webp|bmp|tiff?)$/i;
|
||||
|
||||
/**
|
||||
* Recursively flatten archive tree entries into a list of file paths.
|
||||
*/
|
||||
function flattenEntries(entries, prefix = '') {
|
||||
let paths = [];
|
||||
for (const entry of entries) {
|
||||
const fullPath = entry.path || (prefix ? `${prefix}/${entry.name}` : entry.name);
|
||||
if (entry.is_dir && entry.children) {
|
||||
paths = paths.concat(flattenEntries(entry.children, fullPath));
|
||||
} else if (!entry.is_dir) {
|
||||
paths.push(fullPath);
|
||||
}
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
export function CbzViewer({path}) {
|
||||
const [pages, setPages] = useState(null);
|
||||
const [currentPage, setCurrentPage] = useState(0);
|
||||
const [rtl, setRtl] = useLocalStorage('cbzViewerRtl', false);
|
||||
const [fullscreen, setFullscreen] = useState(false);
|
||||
const [error, setError] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
const load = async () => {
|
||||
try {
|
||||
const contents = await getArchiveContents(path);
|
||||
if (!contents) {
|
||||
setError('Could not read archive contents.');
|
||||
return;
|
||||
}
|
||||
const images = flattenEntries(contents.entries)
|
||||
.filter(name => IMAGE_EXTENSIONS.test(name))
|
||||
.sort();
|
||||
const urls = images.map(member => getArchiveMemberUrl(path, member));
|
||||
setPages(urls);
|
||||
setCurrentPage(0);
|
||||
} catch (e) {
|
||||
setError(e.message || 'Could not read archive contents.');
|
||||
}
|
||||
};
|
||||
load();
|
||||
}, [path]);
|
||||
|
||||
const goNext = useCallback(() => {
|
||||
setCurrentPage(p => Math.min(p + 1, (pages?.length || 1) - 1));
|
||||
}, [pages]);
|
||||
|
||||
const goPrev = useCallback(() => {
|
||||
setCurrentPage(p => Math.max(p - 1, 0));
|
||||
}, []);
|
||||
|
||||
const onLeftClick = rtl ? goNext : goPrev;
|
||||
const onRightClick = rtl ? goPrev : goNext;
|
||||
const imageContainerRef = useRef(null);
|
||||
|
||||
const handleFullscreen = useCallback(() => {
|
||||
setFullscreen(f => !f);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const handleKey = (e) => {
|
||||
if (e.key === 'ArrowLeft') onLeftClick();
|
||||
else if (e.key === 'ArrowRight') onRightClick();
|
||||
else if (e.key === 'Escape') setFullscreen(false);
|
||||
};
|
||||
window.addEventListener('keydown', handleKey);
|
||||
return () => window.removeEventListener('keydown', handleKey);
|
||||
}, [onLeftClick, onRightClick]);
|
||||
|
||||
if (error) {
|
||||
return <Message negative>
|
||||
<Message.Header>Cannot load comic</Message.Header>
|
||||
<p>{error}</p>
|
||||
</Message>;
|
||||
}
|
||||
if (!pages) return <Loader active inline='centered'/>;
|
||||
if (pages.length === 0) {
|
||||
return <Message warning>
|
||||
<Message.Header>No images found</Message.Header>
|
||||
<p>This archive does not contain any image files.</p>
|
||||
</Message>;
|
||||
}
|
||||
|
||||
const hasPrev = currentPage > 0;
|
||||
const hasNext = currentPage < pages.length - 1;
|
||||
|
||||
return <div style={{marginBottom: '1em'}}>
|
||||
<div ref={imageContainerRef} style={{
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
background: '#222',
|
||||
borderRadius: fullscreen ? 0 : '4px',
|
||||
padding: '0.5em',
|
||||
minHeight: '300px',
|
||||
position: fullscreen ? 'fixed' : 'relative',
|
||||
...(fullscreen ? {top: 0, left: 0, right: 0, bottom: 0, zIndex: 9999} : {}),
|
||||
}}>
|
||||
<img
|
||||
src={pages[currentPage]}
|
||||
alt={`Page ${currentPage + 1}`}
|
||||
style={{
|
||||
maxHeight: fullscreen ? '100vh' : '75vh',
|
||||
maxWidth: '100%',
|
||||
objectFit: 'contain',
|
||||
}}
|
||||
/>
|
||||
<div style={{
|
||||
position: 'absolute',
|
||||
top: 0, left: 0, right: 0, bottom: 0,
|
||||
display: 'grid',
|
||||
gridTemplateColumns: '1fr 1fr 1fr',
|
||||
}}>
|
||||
<div style={{cursor: 'pointer'}} onClick={onLeftClick}/>
|
||||
<div style={{cursor: 'pointer'}} onClick={handleFullscreen}/>
|
||||
<div style={{cursor: 'pointer'}} onClick={onRightClick}/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: '0.5em',
|
||||
marginTop: '0.5em',
|
||||
}}>
|
||||
<Button icon basic active={rtl} onClick={() => setRtl(r => !r)}
|
||||
aria-label='Reading direction'>
|
||||
<Icon name={rtl ? 'hand point left' : 'hand point right'}/>
|
||||
</Button>
|
||||
<span style={{flex: 1}}/>
|
||||
<Button icon basic disabled={rtl ? !hasNext : !hasPrev}
|
||||
onClick={onLeftClick} aria-label='Previous page'>
|
||||
<Icon name='chevron left'/>
|
||||
</Button>
|
||||
<Label>{currentPage + 1} / {pages.length}</Label>
|
||||
<Button icon basic disabled={rtl ? !hasPrev : !hasNext}
|
||||
onClick={onRightClick} aria-label='Next page'>
|
||||
<Icon name='chevron right'/>
|
||||
</Button>
|
||||
</div>
|
||||
</div>;
|
||||
}
|
||||
|
|
@ -19,7 +19,7 @@ import {
|
|||
Search,
|
||||
Transition
|
||||
} from 'semantic-ui-react';
|
||||
import {Link, NavLink, useNavigate, useSearchParams} from "react-router";
|
||||
import {Link, NavLink, useNavigate} from "react-router";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {useHotspot, useSearchDirectories, useSearchOrder, useThrottle, useWROLMode} from "../hooks/customHooks";
|
||||
import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts";
|
||||
|
|
@ -701,15 +701,9 @@ export function textEllipsis(str, maxLength = 100, {side = "end", ellipsis = "..
|
|||
}
|
||||
|
||||
export function TabLinks({links}) {
|
||||
const [searchParams] = useSearchParams();
|
||||
|
||||
const getTo = (to) => {
|
||||
return `${to}?${searchParams.toString()}`;
|
||||
}
|
||||
|
||||
return <Menu tabular>
|
||||
{links.map((link) => <NavLink
|
||||
to={getTo(link.to)}
|
||||
to={link.to}
|
||||
style={{padding: '1em'}}
|
||||
key={link.to}
|
||||
end={link.end === true ? true : null}
|
||||
|
|
@ -1084,11 +1078,12 @@ export function emptyToNull(obj) {
|
|||
return obj;
|
||||
}
|
||||
|
||||
export function mimetypeColor(mimetype) {
|
||||
export function mimetypeColor(mimetype, path = '') {
|
||||
if (!mimetype) {
|
||||
return 'grey';
|
||||
}
|
||||
try {
|
||||
const lowerPath = path.toLowerCase();
|
||||
if (mimetype === 'application/pdf') {
|
||||
return 'red'
|
||||
} else if (mimetype.startsWith('video/')) {
|
||||
|
|
@ -1099,6 +1094,11 @@ export function mimetypeColor(mimetype) {
|
|||
return 'green'
|
||||
} else if (mimetype.startsWith('application/epub')) {
|
||||
return 'yellow'
|
||||
} else if (mimetype.includes('cbz') || mimetype.includes('cbr')
|
||||
|| mimetype.includes('comicbook')
|
||||
|| lowerPath.endsWith('.cbz') || lowerPath.endsWith('.cbr')
|
||||
|| lowerPath.endsWith('.cbt') || lowerPath.endsWith('.cb7')) {
|
||||
return 'yellow'
|
||||
} else if (mimetype.startsWith('audio/')) {
|
||||
return 'violet'
|
||||
} else if (isZipMimetype(mimetype)) {
|
||||
|
|
@ -1142,7 +1142,8 @@ export function mimetypeIconName(mimetype, lowerPath = '') {
|
|||
return 'file archive';
|
||||
} else if (mimetype.startsWith('application/x-iso9660-image') || mimetype.startsWith('application/x-raw-disk-image') || mimetype.startsWith('application/x-cd-image')) {
|
||||
return 'dot circle';
|
||||
} else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook') || mimetype.startsWith('application/vnd.amazon.mobi8-ebook')) {
|
||||
} else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook') || mimetype.startsWith('application/vnd.amazon.mobi8-ebook')
|
||||
|| mimetype.startsWith('application/vnd.comicbook') || mimetype.startsWith('application/x-cbz') || mimetype.startsWith('application/x-cbr')) {
|
||||
return 'book';
|
||||
} else if (mimetype.startsWith('text/vtt') || mimetype.startsWith('text/srt')) {
|
||||
return 'closed captioning';
|
||||
|
|
@ -1527,6 +1528,10 @@ export const filterToMimetypes = (filter) => {
|
|||
return ['audio'];
|
||||
} else if (filter === 'ebook') {
|
||||
return ['application/epub+zip', 'application/x-mobipocket-ebook'];
|
||||
} else if (filter === 'doc') {
|
||||
return ['application/epub+zip', 'application/x-mobipocket-ebook', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'application/vnd.oasis.opendocument.text', 'application/x-cbz', 'application/x-cbr'];
|
||||
} else if (filter === 'image') {
|
||||
return ['image'];
|
||||
} else if (filter === 'zip') {
|
||||
|
|
|
|||
532
app/src/components/Docs.js
Normal file
532
app/src/components/Docs.js
Normal file
|
|
@ -0,0 +1,532 @@
|
|||
import React, {useContext, useEffect, useState} from 'react';
|
||||
import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router";
|
||||
import {Grid, Icon as SIcon, Label, Loader, StatisticLabel, StatisticValue} from "semantic-ui-react";
|
||||
import {deleteDocs, getDocStatistics, tagFileGroup, untagFileGroup} from "../api";
|
||||
import {Media, ThemeContext} from "../contexts/contexts";
|
||||
import {
|
||||
APIButton,
|
||||
BackButton,
|
||||
encodeMediaPath,
|
||||
ErrorMessage,
|
||||
humanFileSize,
|
||||
isoDatetimeToAgoPopup,
|
||||
PageContainer,
|
||||
PreviewPath,
|
||||
SearchInput,
|
||||
SortButton,
|
||||
TabLinks,
|
||||
toLocaleString,
|
||||
useTitle
|
||||
} from "./Common";
|
||||
import {Button, darkTheme, Header, Icon, Segment, Statistic, Tab} from "./Theme";
|
||||
import {FilesView} from "./Files";
|
||||
import {useAuthors, useDoc, useOneQuery, useSearchDocs, useSubjects} from "../hooks/customHooks";
|
||||
import {TagsSelector} from "../Tags";
|
||||
import {toast} from "react-semantic-toasts-2";
|
||||
import {CollectionTable} from "./collections/CollectionTable";
|
||||
import {CbzViewer} from "./CbzViewer";
|
||||
import _ from "lodash";
|
||||
|
||||
function DocsPage() {
|
||||
useTitle('Documents');
|
||||
const searchInputRef = React.useRef();
|
||||
|
||||
const {
|
||||
docs,
|
||||
totalPages,
|
||||
activePage,
|
||||
setPage,
|
||||
searchStr,
|
||||
setSearchStr,
|
||||
author,
|
||||
subject,
|
||||
fetchDocs,
|
||||
} = useSearchDocs();
|
||||
|
||||
const [selectedDocs, setSelectedDocs] = useState([]);
|
||||
|
||||
const onSelect = (path, checked) => {
|
||||
if (checked && path) {
|
||||
setSelectedDocs([...selectedDocs, path]);
|
||||
} else if (path) {
|
||||
setSelectedDocs(selectedDocs.filter(i => i !== path));
|
||||
}
|
||||
}
|
||||
|
||||
const onDelete = async () => {
|
||||
const docIds = docs.filter(i => selectedDocs.indexOf(i['primary_path']) >= 0).map(i => i['id']);
|
||||
await deleteDocs(docIds);
|
||||
await fetchDocs();
|
||||
setSelectedDocs([]);
|
||||
}
|
||||
|
||||
const invertSelection = () => {
|
||||
const newSelectedDocs = docs.map(doc => doc['key']).filter(i => selectedDocs.indexOf(i) < 0);
|
||||
setSelectedDocs(newSelectedDocs);
|
||||
}
|
||||
|
||||
const clearSelection = (e) => {
|
||||
if (e) e.preventDefault();
|
||||
setSelectedDocs([]);
|
||||
}
|
||||
|
||||
const selectElm = <div style={{marginTop: '0.5em'}}>
|
||||
<APIButton
|
||||
color='red'
|
||||
disabled={_.isEmpty(selectedDocs)}
|
||||
confirmButton='Delete'
|
||||
confirmContent='Are you sure you want to delete these documents? This cannot be undone.'
|
||||
onClick={onDelete}
|
||||
obeyWROLMode={true}
|
||||
>Delete</APIButton>
|
||||
<Button color='grey' onClick={invertSelection} disabled={_.isEmpty(docs)}>Invert</Button>
|
||||
<Button color='yellow' onClick={clearSelection}
|
||||
disabled={_.isEmpty(docs) || _.isEmpty(selectedDocs)}>Clear</Button>
|
||||
</div>;
|
||||
|
||||
let docOrders = [
|
||||
{value: 'published_datetime', text: 'Published Date', short: 'P.Date'},
|
||||
{value: 'size', text: 'Size'},
|
||||
{value: 'title', text: 'Title'},
|
||||
];
|
||||
if (searchStr) {
|
||||
docOrders = [{value: 'rank', text: 'Rank'}, ...docOrders];
|
||||
}
|
||||
|
||||
const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView(
|
||||
{
|
||||
files: docs,
|
||||
activePage: activePage,
|
||||
totalPages: totalPages,
|
||||
selectElem: selectElm,
|
||||
selectedKeys: selectedDocs,
|
||||
onSelect: onSelect,
|
||||
setPage: setPage,
|
||||
},
|
||||
);
|
||||
|
||||
const [localSearchStr, setLocalSearchStr] = React.useState(searchStr || '');
|
||||
const searchInput = <SearchInput
|
||||
onChange={setLocalSearchStr}
|
||||
onClear={() => setLocalSearchStr(null)}
|
||||
searchStr={localSearchStr}
|
||||
onSubmit={setSearchStr}
|
||||
placeholder='Search Documents...'
|
||||
inputRef={searchInputRef}
|
||||
/>;
|
||||
|
||||
return <>
|
||||
{author && <Header as='h1'>Author: {author}</Header>}
|
||||
{subject && <Header as='h1'>Subject: {subject}</Header>}
|
||||
<Media at='mobile'>
|
||||
<Grid>
|
||||
<Grid.Row>
|
||||
<Grid.Column width={2}>{selectButton}</Grid.Column>
|
||||
<Grid.Column width={2}>{viewButton}</Grid.Column>
|
||||
<Grid.Column width={4}>{limitDropdown}</Grid.Column>
|
||||
<Grid.Column width={2}>{tagQuerySelector}</Grid.Column>
|
||||
<Grid.Column width={6}><SortButton sorts={docOrders}/></Grid.Column>
|
||||
</Grid.Row>
|
||||
<Grid.Row width={16}>
|
||||
<Grid.Column>{searchInput}</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
</Media>
|
||||
<Media greaterThanOrEqual='tablet'>
|
||||
<Grid>
|
||||
<Grid.Row>
|
||||
<Grid.Column width={1}>{selectButton}</Grid.Column>
|
||||
<Grid.Column width={1}>{viewButton}</Grid.Column>
|
||||
<Grid.Column width={2}>{limitDropdown}</Grid.Column>
|
||||
<Grid.Column width={1}>{tagQuerySelector}</Grid.Column>
|
||||
<Grid.Column width={3}><SortButton sorts={docOrders}/></Grid.Column>
|
||||
<Grid.Column width={8}>{searchInput}</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
</Media>
|
||||
{body}
|
||||
{paginator}
|
||||
</>;
|
||||
}
|
||||
|
||||
function DocPage() {
|
||||
const {fileGroupId} = useParams();
|
||||
const navigate = useNavigate();
|
||||
const {docFile, doc, fetchDoc} = useDoc(parseInt(fileGroupId));
|
||||
const {theme} = useContext(ThemeContext);
|
||||
|
||||
const title = docFile ? (docFile.title || docFile.name) : null;
|
||||
useTitle(title);
|
||||
|
||||
if (docFile === null) {
|
||||
return <Segment><Loader active/></Segment>;
|
||||
}
|
||||
if (docFile === undefined) {
|
||||
return <>
|
||||
<Header as='h2'>Unknown document</Header>
|
||||
<Header as='h4'>This document does not exist</Header>
|
||||
</>;
|
||||
}
|
||||
|
||||
const {data, size, directory} = docFile;
|
||||
|
||||
// Cover image.
|
||||
const coverPath = data && data.cover_path ? data.cover_path : null;
|
||||
const coverUrl = coverPath ? `/media/${encodeMediaPath(coverPath)}` : null;
|
||||
|
||||
// Primary file URLs.
|
||||
const primaryPath = docFile.primary_path;
|
||||
const downloadUrl = primaryPath ? `/download/${encodeMediaPath(primaryPath)}` : null;
|
||||
const isEpub = docFile.mimetype && docFile.mimetype.startsWith('application/epub');
|
||||
const mt = docFile.mimetype || '';
|
||||
const primaryExt = (primaryPath || '').toLowerCase().split('.').pop();
|
||||
const isCbz = mt.includes('cbz') || mt.includes('cbr') || mt.includes('comicbook+zip') || mt.includes('comicbook-rar')
|
||||
|| ['cbz', 'cbr', 'cbt', 'cb7'].includes(primaryExt);
|
||||
const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
|
||||
const openUrl = viewerUrl || (primaryPath ? `/media/${encodeMediaPath(primaryPath)}` : null);
|
||||
|
||||
const handleDelete = async () => {
|
||||
if (doc && doc.id) {
|
||||
await deleteDocs([doc.id]);
|
||||
toast({
|
||||
type: 'success',
|
||||
title: 'Document Deleted',
|
||||
description: 'Document was successfully deleted.',
|
||||
time: 2000,
|
||||
});
|
||||
navigate(-1);
|
||||
}
|
||||
};
|
||||
|
||||
const localAddTag = async (name) => {
|
||||
await tagFileGroup(docFile, name);
|
||||
await fetchDoc();
|
||||
};
|
||||
|
||||
const localRemoveTag = async (name) => {
|
||||
await untagFileGroup(docFile, name);
|
||||
await fetchDoc();
|
||||
};
|
||||
|
||||
const publishedDatetimeString = docFile.published_datetime
|
||||
? isoDatetimeToAgoPopup(docFile.published_datetime, true)
|
||||
: null;
|
||||
|
||||
// Format badge based on mimetype.
|
||||
const formatLabel = () => {
|
||||
const mt = docFile.mimetype || '';
|
||||
if (mt.startsWith('application/epub')) return 'EPUB';
|
||||
if (mt === 'application/x-mobipocket-ebook') return 'MOBI';
|
||||
if (mt === 'application/pdf') return 'PDF';
|
||||
if (mt.includes('wordprocessingml')) return 'DOCX';
|
||||
if (mt === 'application/msword') return 'DOC';
|
||||
if (mt.includes('opendocument.text')) return 'ODT';
|
||||
if (mt.includes('cbz') || mt.includes('comicbook+zip')) return 'CBZ';
|
||||
if (mt.includes('cbr') || mt.includes('comicbook-rar')) return 'CBR';
|
||||
if (mt.includes('cbt')) return 'CBT';
|
||||
if (mt.includes('cb7') || mt.includes('x-7z')) return 'CB7';
|
||||
return mt.split('/').pop().toUpperCase();
|
||||
};
|
||||
|
||||
// About pane.
|
||||
const aboutPane = {
|
||||
menuItem: 'About', render: () => <Tab.Pane>
|
||||
{doc && doc.description && <>
|
||||
<Header as='h3'>Description</Header>
|
||||
<p>{doc.description}</p>
|
||||
</>}
|
||||
|
||||
{doc && doc.publisher && <>
|
||||
<Header as='h3'>Publisher</Header>
|
||||
<p>{doc.publisher}</p>
|
||||
</>}
|
||||
|
||||
{doc && doc.language && <>
|
||||
<Header as='h3'>Language</Header>
|
||||
<p>{doc.language}</p>
|
||||
</>}
|
||||
|
||||
{doc && doc.subject && <>
|
||||
<Header as='h3'>Subject</Header>
|
||||
<p><Link to={`/docs?subject=${encodeURIComponent(doc.subject)}`}>{doc.subject}</Link></p>
|
||||
</>}
|
||||
|
||||
{doc && doc.page_count && <>
|
||||
<Header as='h3'>Page Count</Header>
|
||||
<p>{doc.page_count}</p>
|
||||
</>}
|
||||
|
||||
<Header as='h3'>Size</Header>
|
||||
<p>{humanFileSize(size)}</p>
|
||||
</Tab.Pane>
|
||||
};
|
||||
|
||||
// Files pane.
|
||||
const filesPane = {
|
||||
menuItem: 'Files', render: () => <Tab.Pane>
|
||||
{docFile.files && docFile.files.map((file, idx) =>
|
||||
<div key={idx} style={{marginBottom: '0.5em'}}>
|
||||
<PreviewPath path={file.path} mimetype={file.mimetype} taggable={false}>
|
||||
{file.path}
|
||||
</PreviewPath>
|
||||
{file.size && <span style={{marginLeft: '1em', color: 'grey'}}>
|
||||
({humanFileSize(file.size)})
|
||||
</span>}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<Header as='h3'>Directory</Header>
|
||||
<p>{directory || 'Unknown'}</p>
|
||||
</Tab.Pane>
|
||||
};
|
||||
|
||||
const tabPanes = [aboutPane, filesPane];
|
||||
const tabMenu = theme === darkTheme ? {inverted: true, attached: true} : {attached: true};
|
||||
|
||||
// Inline reader: epub viewer iframe or PDF iframe.
|
||||
const isPdf = docFile.mimetype === 'application/pdf';
|
||||
const canEmbed = isEpub || isPdf;
|
||||
const embedUrl = isEpub ? viewerUrl : (isPdf && primaryPath ? `/media/${encodeMediaPath(primaryPath)}` : null);
|
||||
|
||||
return <>
|
||||
<BackButton/>
|
||||
|
||||
{isCbz && primaryPath && <CbzViewer path={primaryPath}/>}
|
||||
|
||||
{canEmbed && embedUrl && !isCbz && <div style={{marginBottom: '1em'}}>
|
||||
<iframe
|
||||
src={embedUrl}
|
||||
title={docFile.title || docFile.name}
|
||||
style={{width: '100%', height: '80vh', border: '1px solid #ccc', borderRadius: '4px'}}
|
||||
/>
|
||||
</div>}
|
||||
|
||||
<Segment>
|
||||
<Header as='h2'>{docFile.title || docFile.name}</Header>
|
||||
|
||||
{docFile.author && <Header as='h3'>
|
||||
Author: <Link to={`/docs?author=${encodeURIComponent(docFile.author)}`}>{docFile.author}</Link>
|
||||
</Header>}
|
||||
|
||||
<Grid columns={2} stackable>
|
||||
<Grid.Row>
|
||||
{publishedDatetimeString && <Grid.Column>
|
||||
<Header as='h4'>Published: {publishedDatetimeString}</Header>
|
||||
</Grid.Column>}
|
||||
<Grid.Column>
|
||||
<Label>{formatLabel()}</Label>
|
||||
{doc && doc.page_count && <Label>{doc.page_count} pages</Label>}
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
|
||||
<div style={{marginTop: '1em'}}>
|
||||
{openUrl && <Button as='a' href={openUrl} target='_blank' rel='noreferrer' color='violet'>
|
||||
<SIcon name='expand arrows alternate'/> Open
|
||||
</Button>}
|
||||
{downloadUrl && <Button as='a' href={downloadUrl}>
|
||||
<SIcon name='download'/> Download
|
||||
</Button>}
|
||||
<APIButton
|
||||
color='red'
|
||||
confirmContent='Are you sure you want to delete this document? All files will be deleted.'
|
||||
confirmButton='Delete'
|
||||
onClick={handleDelete}
|
||||
obeyWROLMode={true}
|
||||
>Delete</APIButton>
|
||||
</div>
|
||||
</Segment>
|
||||
|
||||
<Segment>
|
||||
<TagsSelector selectedTagNames={docFile.tags} onAdd={localAddTag} onRemove={localRemoveTag}/>
|
||||
</Segment>
|
||||
|
||||
<Tab menu={tabMenu} panes={tabPanes}/>
|
||||
</>;
|
||||
}
|
||||
|
||||
const AUTHOR_COLUMNS = [
|
||||
{key: 'name', label: 'Author', sortable: true, width: 10},
|
||||
{key: 'item_count', label: 'Documents', sortable: true, align: 'right', width: 6},
|
||||
];
|
||||
|
||||
const AUTHOR_ROUTES = {
|
||||
search: '/docs',
|
||||
searchParam: 'author',
|
||||
};
|
||||
|
||||
const SUBJECT_COLUMNS = [
|
||||
{key: 'name', label: 'Subject', sortable: true, width: 10},
|
||||
{key: 'item_count', label: 'Documents', sortable: true, align: 'right', width: 6},
|
||||
];
|
||||
|
||||
const SUBJECT_ROUTES = {
|
||||
search: '/docs',
|
||||
searchParam: 'subject',
|
||||
};
|
||||
|
||||
function AuthorsPage() {
|
||||
useTitle('Doc Authors');
|
||||
|
||||
const [authors] = useAuthors();
|
||||
const [searchStr, setSearchStr] = useOneQuery('author');
|
||||
const searchInputRef = React.useRef();
|
||||
|
||||
const header = <div style={{marginBottom: '1em'}}>
|
||||
<Grid stackable columns={2}>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<SearchInput
|
||||
placeholder='Author filter...'
|
||||
size='large'
|
||||
searchStr={searchStr}
|
||||
disabled={!Array.isArray(authors) || authors.length === 0}
|
||||
onClear={() => setSearchStr('')}
|
||||
onChange={setSearchStr}
|
||||
onSubmit={null}
|
||||
inputRef={searchInputRef}
|
||||
/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
</div>;
|
||||
|
||||
return <>
|
||||
{header}
|
||||
<CollectionTable
|
||||
collections={authors}
|
||||
columns={AUTHOR_COLUMNS}
|
||||
routes={AUTHOR_ROUTES}
|
||||
searchStr={searchStr}
|
||||
emptyMessage='No authors found. Index some documents with author metadata!'
|
||||
/>
|
||||
</>;
|
||||
}
|
||||
|
||||
function SubjectsPage() {
|
||||
useTitle('Doc Subjects');
|
||||
|
||||
const [subjects] = useSubjects();
|
||||
const [searchStr, setSearchStr] = useOneQuery('subject');
|
||||
const searchInputRef = React.useRef();
|
||||
|
||||
const header = <div style={{marginBottom: '1em'}}>
|
||||
<Grid stackable columns={2}>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<SearchInput
|
||||
placeholder='Subject filter...'
|
||||
size='large'
|
||||
searchStr={searchStr}
|
||||
disabled={!Array.isArray(subjects) || subjects.length === 0}
|
||||
onClear={() => setSearchStr('')}
|
||||
onChange={setSearchStr}
|
||||
onSubmit={null}
|
||||
inputRef={searchInputRef}
|
||||
/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
</div>;
|
||||
|
||||
return <>
|
||||
{header}
|
||||
<CollectionTable
|
||||
collections={subjects}
|
||||
columns={SUBJECT_COLUMNS}
|
||||
routes={SUBJECT_ROUTES}
|
||||
searchStr={searchStr}
|
||||
emptyMessage='No subjects found. Index some documents with subject metadata!'
|
||||
/>
|
||||
</>;
|
||||
}
|
||||
|
||||
function DocsStatisticsPage() {
|
||||
useTitle('Doc Statistics');
|
||||
|
||||
const [statistics, setStatistics] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
const fetchStats = async () => {
|
||||
try {
|
||||
const response = await getDocStatistics();
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setStatistics(data.statistics);
|
||||
}
|
||||
} catch (e) {
|
||||
setStatistics(undefined);
|
||||
}
|
||||
};
|
||||
fetchStats();
|
||||
}, []);
|
||||
|
||||
if (statistics === null) {
|
||||
return <Loader active inline='centered'/>;
|
||||
}
|
||||
if (statistics === undefined) {
|
||||
return <ErrorMessage>Unable to fetch Doc Statistics</ErrorMessage>;
|
||||
}
|
||||
|
||||
return <Segment>
|
||||
<Header as='h1' textAlign='center'>Documents</Header>
|
||||
<Statistic.Group>
|
||||
<Statistic style={{margin: '2em'}}>
|
||||
<StatisticValue>{toLocaleString(statistics.doc_count)}</StatisticValue>
|
||||
<StatisticLabel>Documents</StatisticLabel>
|
||||
</Statistic>
|
||||
<Statistic style={{margin: '2em'}}>
|
||||
<StatisticValue>{toLocaleString(statistics.epub_count)}</StatisticValue>
|
||||
<StatisticLabel>eBooks</StatisticLabel>
|
||||
</Statistic>
|
||||
<Statistic style={{margin: '2em'}}>
|
||||
<StatisticValue>{toLocaleString(statistics.pdf_count)}</StatisticValue>
|
||||
<StatisticLabel>PDFs</StatisticLabel>
|
||||
</Statistic>
|
||||
<Statistic style={{margin: '2em'}}>
|
||||
<StatisticValue>{toLocaleString(statistics.other_count)}</StatisticValue>
|
||||
<StatisticLabel>Other</StatisticLabel>
|
||||
</Statistic>
|
||||
<Statistic style={{margin: '2em'}}>
|
||||
<StatisticValue>{humanFileSize(statistics.total_size)}</StatisticValue>
|
||||
<StatisticLabel>Total Size</StatisticLabel>
|
||||
</Statistic>
|
||||
<Statistic style={{margin: '2em'}}>
|
||||
<StatisticValue>{toLocaleString(statistics.author_count)}</StatisticValue>
|
||||
<StatisticLabel>Authors</StatisticLabel>
|
||||
</Statistic>
|
||||
<Statistic style={{margin: '2em'}}>
|
||||
<StatisticValue>{toLocaleString(statistics.subject_count)}</StatisticValue>
|
||||
<StatisticLabel>Subjects</StatisticLabel>
|
||||
</Statistic>
|
||||
</Statistic.Group>
|
||||
</Segment>;
|
||||
}
|
||||
|
||||
export function DocsRoute() {
|
||||
const location = useLocation();
|
||||
const path = location.pathname;
|
||||
|
||||
const links = [
|
||||
{
|
||||
text: 'Docs',
|
||||
to: '/docs',
|
||||
end: true,
|
||||
isActive: () => path === '/docs' || /^\/docs\/view\/\d+$/.test(path)
|
||||
},
|
||||
{text: 'Subjects', to: '/docs/subjects', isActive: () => path.startsWith('/docs/subjects')},
|
||||
{text: 'Authors', to: '/docs/authors', isActive: () => path.startsWith('/docs/authors')},
|
||||
{text: 'Statistics', to: '/docs/statistics'},
|
||||
];
|
||||
|
||||
return <PageContainer>
|
||||
<TabLinks links={links}/>
|
||||
<Routes>
|
||||
<Route path='/' element={<DocsPage/>}/>
|
||||
<Route path='view/:fileGroupId' element={<DocPage/>}/>
|
||||
<Route path='subjects' element={<SubjectsPage/>}/>
|
||||
<Route path='authors' element={<AuthorsPage/>}/>
|
||||
<Route path='statistics' element={<DocsStatisticsPage/>}/>
|
||||
</Routes>
|
||||
</PageContainer>;
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import {Media} from "../contexts/contexts";
|
|||
import {encodeMediaPath, isSupportedArchive} from "./Common";
|
||||
import {getFile, tagFileGroup, untagFileGroup} from "../api";
|
||||
import {ArchivePreviewContent} from "./ArchivePreview";
|
||||
import {CbzViewer} from "./CbzViewer";
|
||||
import {StlViewer} from "react-stl-viewer";
|
||||
import {Button, Modal} from "./Theme";
|
||||
import {toast} from "react-semantic-toasts-2";
|
||||
|
|
@ -361,6 +362,9 @@ export function FilePreviewProvider({children}) {
|
|||
setModalContent(getAudioPreviewModal(previewFile), url, downloadURL, path, taggable);
|
||||
} else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.stl')) {
|
||||
setModalContent(getSTLPreviewModal(previewFile), null, downloadURL, path, taggable);
|
||||
} else if (mimetype.includes('cbz') || mimetype.includes('cbr') || mimetype.includes('comicbook+zip') || mimetype.includes('comicbook-rar')
|
||||
|| lowerPath.endsWith('.cbz') || lowerPath.endsWith('.cbr') || lowerPath.endsWith('.cbt') || lowerPath.endsWith('.cb7')) {
|
||||
setModalContent(<Modal.Content><CbzViewer path={path}/></Modal.Content>, null, downloadURL, path, taggable);
|
||||
} else if (isSupportedArchive(mimetype, lowerPath)) {
|
||||
setModalContent(<ArchivePreviewContent previewFile={previewFile}/>, null, downloadURL, path, taggable);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -37,12 +37,13 @@ import {
|
|||
usePages,
|
||||
useSearchFiles,
|
||||
useSearchFilter,
|
||||
useSearchOrder,
|
||||
useSearchView,
|
||||
useStatusFlag,
|
||||
useWROLMode
|
||||
} from "../hooks/customHooks";
|
||||
import {useFileWorkerStatus} from "../contexts/FileWorkerStatusContext";
|
||||
import {Route, Routes} from "react-router";
|
||||
import {Link, Route, Routes} from "react-router";
|
||||
import {CardPlaceholder} from "./Placeholder";
|
||||
import {ArchiveCard, ArchiveRowCells} from "./Archive";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
|
|
@ -66,20 +67,23 @@ import {Headlines} from "./Headline";
|
|||
import {useSearch} from "./Search";
|
||||
import {FILES_MEDIA_URI} from "./Vars";
|
||||
|
||||
function EbookCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
function EbookCard({file, sortField}) {
|
||||
const {s, t} = useContext(ThemeContext);
|
||||
|
||||
const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
|
||||
const isEpub = file['mimetype'].startsWith('application/epub');
|
||||
const isEpub = file['mimetype'] && file['mimetype'].startsWith('application/epub');
|
||||
const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
|
||||
|
||||
const color = mimetypeColor(file.mimetype);
|
||||
// Link to doc detail page if this is a modeled doc.
|
||||
const detailUrl = file.model === 'doc' ? `/docs/view/${file.id}` : null;
|
||||
|
||||
const color = mimetypeColor(file.mimetype, file.primary_path);
|
||||
const title = file.title || file.stem || file.name;
|
||||
const header = <ExternalCardLink to={viewerUrl || downloadUrl} className='card-title-ellipsis'>
|
||||
{title}
|
||||
</ExternalCardLink>;
|
||||
const header = detailUrl
|
||||
? <Link to={detailUrl} className='no-link-underscore card-link card-title-ellipsis' {...t}>{title}</Link>
|
||||
: <ExternalCardLink to={viewerUrl || downloadUrl} className='card-title-ellipsis'>{title}</ExternalCardLink>;
|
||||
return <Card color={color}>
|
||||
<CardPoster file={file} preview={true}/>
|
||||
<CardPoster file={file} to={detailUrl}/>
|
||||
<CardContent {...s}>
|
||||
<CardHeader>
|
||||
<Container textAlign='left'>
|
||||
|
|
@ -90,7 +94,10 @@ function EbookCard({file}) {
|
|||
</CardHeader>
|
||||
<CardMeta>
|
||||
{file.author ? <b {...s}>{file.author}</b> : null}
|
||||
{file.size && <p {...s}>{humanFileSize(file.size)}</p>}
|
||||
{sortField === 'published_datetime'
|
||||
? <p {...s}>{file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime, false) : null}</p>
|
||||
: file.size && <p {...s}>{humanFileSize(file.size)}</p>
|
||||
}
|
||||
</CardMeta>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
|
@ -128,10 +135,14 @@ function ImageCard({file}) {
|
|||
|
||||
function FileCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
const {sort} = useSearchOrder();
|
||||
const sortField = sort ? sort.replace(/^-/, '') : null;
|
||||
|
||||
const isEbookType = file.mimetype && (
|
||||
const isDocType = file.model === 'doc' || (file.mimetype && (
|
||||
file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
|
||||
);
|
||||
));
|
||||
// Default doc sort is published_datetime when no order query param is set.
|
||||
const docSortField = sortField || 'published_datetime';
|
||||
|
||||
if (file.model === 'video' && 'video' in file) {
|
||||
return <VideoCard key={file['primary_path']} file={file}/>;
|
||||
|
|
@ -139,12 +150,12 @@ function FileCard({file}) {
|
|||
return <ArchiveCard key={file['primary_path']} file={file}/>;
|
||||
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
|
||||
return <ImageCard key={file['primary_path']} file={file}/>;
|
||||
} else if (isEbookType) {
|
||||
return <EbookCard key={file['primary_path']} file={file}/>;
|
||||
} else if (isDocType) {
|
||||
return <EbookCard key={file['primary_path']} file={file} sortField={docSortField}/>;
|
||||
}
|
||||
|
||||
const author = file.author;
|
||||
const color = mimetypeColor(file.mimetype);
|
||||
const color = mimetypeColor(file.mimetype, file.primary_path);
|
||||
const size = file.size !== null && file.size !== undefined ? humanFileSize(file.size) : null;
|
||||
|
||||
const title = file.title || file.name || file.primary_path;
|
||||
|
|
@ -160,8 +171,13 @@ function FileCard({file}) {
|
|||
</PreviewLink>
|
||||
</CardHeader>
|
||||
{author && <b {...s}>{author}</b>}
|
||||
<p>{isoDatetimeToAgoPopup(dt, false)}</p>
|
||||
<p>{size}</p>
|
||||
{sortField === 'published_datetime'
|
||||
? <p>{file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime, false) : null}</p>
|
||||
: <>
|
||||
<p>{isoDatetimeToAgoPopup(dt, false)}</p>
|
||||
<p>{size}</p>
|
||||
</>
|
||||
}
|
||||
</CardContent>
|
||||
</Card>
|
||||
}
|
||||
|
|
@ -206,7 +222,7 @@ function ImageRowCells({file}) {
|
|||
}
|
||||
|
||||
export function EbookRowCells({file}) {
|
||||
let cover = <Card.Icon><FileIcon file={file}/></Card.Icon>;
|
||||
let cover = <FileIcon file={file} size='large'/>;
|
||||
const posterPath = findPosterPath(file);
|
||||
if (posterPath) {
|
||||
const coverSrc = `/media/${encodeMediaPath(posterPath)}`;
|
||||
|
|
@ -229,9 +245,9 @@ export function EbookRowCells({file}) {
|
|||
}
|
||||
|
||||
function FileRow({file}) {
|
||||
const isEbookType = file.mimetype && (
|
||||
const isDocType = file.model === 'doc' || (file.mimetype && (
|
||||
file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
|
||||
);
|
||||
));
|
||||
|
||||
if (file.model === 'video' && 'video' in file) {
|
||||
return <VideoRowCells file={file}/>;
|
||||
|
|
@ -239,7 +255,7 @@ function FileRow({file}) {
|
|||
return <ArchiveRowCells file={file}/>;
|
||||
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
|
||||
return <ImageRowCells file={file}/>;
|
||||
} else if (isEbookType) {
|
||||
} else if (isDocType) {
|
||||
return <EbookRowCells key={file['primary_path']} file={file}/>;
|
||||
}
|
||||
|
||||
|
|
@ -480,6 +496,7 @@ export function FileSearchFilterButton({size = 'medium'}) {
|
|||
{key: 'archive', text: 'Archive', value: 'archive'},
|
||||
{key: 'pdf', text: 'PDF', value: 'pdf'},
|
||||
{key: 'ebook', text: 'eBook', value: 'ebook'},
|
||||
{key: 'doc', text: 'Document', value: 'doc'},
|
||||
{key: 'audio', text: 'Audio', value: 'audio'},
|
||||
{key: 'image', text: 'Image', value: 'image'},
|
||||
{key: 'zip', text: 'ZIP', value: 'zip'},
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ function updateFavicon(colorName) {
|
|||
const links = [
|
||||
{text: 'Videos', to: '/videos', key: 'videos'},
|
||||
{text: 'Archive', to: '/archive', key: 'archive'},
|
||||
{text: 'Docs', to: '/docs', key: 'docs'},
|
||||
{text: 'Map', to: '/map', key: 'map'},
|
||||
{text: 'Zim', to: '/zim', key: 'zim'},
|
||||
{text: 'Files', to: '/files', key: 'files'},
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
|
|||
const pages = usePages(defaultLimit, totalPages);
|
||||
// text/html, video*, image*, etc.
|
||||
const filter = searchParams.get('filter');
|
||||
// archive/video/ebook/etc.
|
||||
// archive/video/doc/etc.
|
||||
const model_ = searchParams.get('model') || model;
|
||||
|
||||
const anySearch = (!datesIsEmpty) || searchStr || (activeTags && activeTags.length > 0) || filter || model;
|
||||
|
|
@ -121,6 +121,8 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
fileGroups: [],
|
||||
channels: [],
|
||||
domains: [],
|
||||
authors: [],
|
||||
subjects: [],
|
||||
zimsEstimates: [],
|
||||
}
|
||||
const [suggestions, setSuggestions] = React.useState(defaultSuggestions);
|
||||
|
|
@ -162,6 +164,8 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
...prevState,
|
||||
channels: generalData.channels,
|
||||
domains: generalData.domains,
|
||||
authors: generalData.authors,
|
||||
subjects: generalData.subjects,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -274,6 +278,30 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
})
|
||||
}
|
||||
}
|
||||
if (newSuggestions.authors && newSuggestions.authors.length > 0) {
|
||||
results.authors = {
|
||||
name: 'Authors', results: newSuggestions.authors.map(i => {
|
||||
return {
|
||||
type: 'author',
|
||||
title: i.name,
|
||||
id: i.id,
|
||||
location: `/docs?author=${encodeURIComponent(i.name)}`
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
if (newSuggestions.subjects && newSuggestions.subjects.length > 0) {
|
||||
results.subjects = {
|
||||
name: 'Subjects', results: newSuggestions.subjects.map(i => {
|
||||
return {
|
||||
type: 'subject',
|
||||
title: i.name,
|
||||
id: i.id,
|
||||
location: `/docs?subject=${encodeURIComponent(i.name)}`
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Match at most 5 Tags.
|
||||
const matchingTags = searchStr ? fuzzyMatchTagsByName(searchStr).slice(0, 5) : null;
|
||||
|
|
@ -304,6 +332,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
otherSum: otherSum,
|
||||
channels: newSuggestions.channels.length,
|
||||
domains: newSuggestions.domains.length,
|
||||
authors: newSuggestions.authors?.length,
|
||||
subjects: newSuggestions.subjects?.length,
|
||||
tags: matchingTags?.length,
|
||||
apps: matchingApps?.length,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -854,6 +854,8 @@ export function VideosRoute(props) {
|
|||
export function VideoCard({file}) {
|
||||
const {video} = file;
|
||||
const {s} = useContext(ThemeContext);
|
||||
const {sort} = useSearchOrder();
|
||||
const sortField = sort ? sort.replace(/^-/, '') : null;
|
||||
|
||||
// Default to video FilePreview for lone video files.
|
||||
let video_url;
|
||||
|
|
@ -904,7 +906,18 @@ export function VideoCard({file}) {
|
|||
{channel && <Link to={channel_url} className="no-link-underscore card-link">
|
||||
<b {...s}>{channel.name}</b>
|
||||
</Link>}
|
||||
<p {...s}>{isoDatetimeToAgoPopup(file.published_datetime, false)}</p>
|
||||
{sortField === 'length'
|
||||
? <p {...s}>{secondsToFullDuration(file.length || 0)}</p>
|
||||
: sortField === 'size'
|
||||
? <p {...s}>{humanFileSize(file.size)}</p>
|
||||
: sortField === 'view_count'
|
||||
? <p {...s}>{humanNumber(video.view_count || 0)} views</p>
|
||||
: sortField === 'viewed'
|
||||
? <p {...s}>{isoDatetimeToAgoPopup(file.viewed, false)}</p>
|
||||
: sortField === 'download_datetime'
|
||||
? <p {...s}>{isoDatetimeToAgoPopup(file.download_datetime, false)}</p>
|
||||
: <p {...s}>{isoDatetimeToAgoPopup(file.published_datetime, false)}</p>
|
||||
}
|
||||
</Container>
|
||||
</CardDescription>
|
||||
</CardContent>
|
||||
|
|
|
|||
|
|
@ -4,9 +4,12 @@ import {
|
|||
createChannel,
|
||||
fetchChannels,
|
||||
fetchDecoded,
|
||||
fetchAuthors,
|
||||
fetchDomains,
|
||||
fetchSubjects,
|
||||
filesSearch,
|
||||
getArchive,
|
||||
getDocByFileGroup,
|
||||
getArchiveStatistics,
|
||||
getChannel,
|
||||
getConfigs,
|
||||
|
|
@ -28,6 +31,7 @@ import {
|
|||
searchArchives,
|
||||
searchChannels,
|
||||
searchDirectories,
|
||||
searchDocs,
|
||||
searchVideos,
|
||||
searchZim,
|
||||
updateChannel,
|
||||
|
|
@ -294,6 +298,81 @@ export const useDomains = () => {
|
|||
return [domains, total];
|
||||
}
|
||||
|
||||
export const useAuthors = () => {
|
||||
const [authors, setAuthors] = useState(null);
|
||||
const [total, setTotal] = useState(null);
|
||||
|
||||
const _fetchAuthors = async () => {
|
||||
setAuthors(null);
|
||||
setTotal(0);
|
||||
try {
|
||||
let [authors, total] = await fetchAuthors();
|
||||
setAuthors(authors);
|
||||
setTotal(total);
|
||||
} catch (e) {
|
||||
setAuthors(undefined);
|
||||
setTotal(0);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
_fetchAuthors();
|
||||
}, []);
|
||||
|
||||
return [authors, total];
|
||||
}
|
||||
|
||||
export const useSubjects = () => {
|
||||
const [subjects, setSubjects] = useState(null);
|
||||
const [total, setTotal] = useState(null);
|
||||
|
||||
const _fetchSubjects = async () => {
|
||||
setSubjects(null);
|
||||
setTotal(0);
|
||||
try {
|
||||
let [subjects, total] = await fetchSubjects();
|
||||
setSubjects(subjects);
|
||||
setTotal(total);
|
||||
} catch (e) {
|
||||
setSubjects(undefined);
|
||||
setTotal(0);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
_fetchSubjects();
|
||||
}, []);
|
||||
|
||||
return [subjects, total];
|
||||
}
|
||||
|
||||
export const useDoc = (fileGroupId) => {
|
||||
const [docFile, setDocFile] = useState(null);
|
||||
const [doc, setDoc] = useState(null);
|
||||
|
||||
const fetchDoc = async () => {
|
||||
try {
|
||||
const response = await getDocByFileGroup(fileGroupId);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
setDocFile(data.file_group);
|
||||
setDoc(data.doc);
|
||||
} else {
|
||||
setDocFile(undefined);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
setDocFile(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
fetchDoc();
|
||||
}, [fileGroupId]);
|
||||
|
||||
return {docFile, doc, fetchDoc};
|
||||
}
|
||||
|
||||
export const useDomain = (domainId) => {
|
||||
const emptyDomain = {
|
||||
domain: '',
|
||||
|
|
@ -455,6 +534,62 @@ export const useSearchArchives = (defaultLimit) => {
|
|||
}
|
||||
}
|
||||
|
||||
export const useSearchDocs = (defaultLimit) => {
|
||||
const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
|
||||
const {searchParams, updateQuery} = React.useContext(QueryContext);
|
||||
const searchStr = searchParams.get('q') || '';
|
||||
const order = searchParams.get('order');
|
||||
const activeTags = searchParams.getAll('tag');
|
||||
const author = searchParams.get('author') || '';
|
||||
const subject = searchParams.get('subject') || '';
|
||||
|
||||
const [docs, setDocs] = useState(null);
|
||||
const [totalPages, setTotalPages] = useState(0);
|
||||
|
||||
const localSearchDocs = async () => {
|
||||
setDocs(null);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject);
|
||||
setDocs(docs_);
|
||||
setTotalPages(calculateTotalPages(total, limit));
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
setDocs(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
localSearchDocs();
|
||||
}, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject]);
|
||||
|
||||
const setSearchStr = (value) => {
|
||||
updateQuery({q: value, o: 0, order: undefined});
|
||||
}
|
||||
|
||||
const setOrderBy = (value) => {
|
||||
setPage(1);
|
||||
updateQuery({order: value});
|
||||
}
|
||||
|
||||
return {
|
||||
docs,
|
||||
limit,
|
||||
setLimit,
|
||||
offset,
|
||||
order,
|
||||
setOrderBy,
|
||||
totalPages,
|
||||
activePage,
|
||||
setPage,
|
||||
searchStr,
|
||||
setSearchStr,
|
||||
author,
|
||||
subject,
|
||||
fetchDocs: localSearchDocs,
|
||||
}
|
||||
}
|
||||
|
||||
export const useSearchVideos = (defaultLimit, channelId, order_by) => {
|
||||
const {searchParams, updateQuery} = React.useContext(QueryContext);
|
||||
const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ sysstat
|
|||
tar
|
||||
tcpdump
|
||||
tmux
|
||||
unrar-free
|
||||
unzip
|
||||
vim
|
||||
vlc
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ ENV DOCKER=true
|
|||
WORKDIR /opt/wrolpi
|
||||
|
||||
# Update dependencies for the services install.
|
||||
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unzip
|
||||
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip
|
||||
RUN ffmpeg -version
|
||||
|
||||
# Install Deno runtime for yt-dlp
|
||||
|
|
|
|||
|
|
@ -59,13 +59,6 @@
|
|||
reverse_proxy api:8081
|
||||
}
|
||||
|
||||
# API docs.
|
||||
handle /docs/* {
|
||||
reverse_proxy api:8081
|
||||
}
|
||||
handle /docs {
|
||||
redir * /docs/ 301
|
||||
}
|
||||
|
||||
# Custom MIME types not in Caddy's defaults.
|
||||
@yaml path *.yaml *.yml
|
||||
|
|
|
|||
|
|
@ -59,13 +59,6 @@
|
|||
reverse_proxy 0.0.0.0:8081
|
||||
}
|
||||
|
||||
# API docs.
|
||||
handle /docs/* {
|
||||
reverse_proxy 0.0.0.0:8081
|
||||
}
|
||||
handle /docs {
|
||||
redir * /docs/ 301
|
||||
}
|
||||
|
||||
# Custom MIME types not in Caddy's defaults.
|
||||
@yaml path *.yaml *.yml
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from .archive import api
|
||||
from .docs import api
|
||||
from .inventory import api
|
||||
from .map import api
|
||||
from .otp import api
|
||||
|
|
|
|||
188
modules/docs/__init__.py
Normal file
188
modules/docs/__init__.py
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
import asyncio
|
||||
import logging
|
||||
from typing import Callable, List, Tuple
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import register_modeler, register_refresh_cleanup, \
|
||||
truncate_object_bytes, split_lines_by_length
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.indexers import Indexer
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
|
||||
from .extractors import extract_metadata
|
||||
from .lib import get_or_create_subject_collection, get_or_create_author_collection, is_valid_author, split_authors, \
|
||||
normalize_author, normalize_subject, split_subjects, is_valid_subject, discover_calibre_cover
|
||||
from .models import Doc, DOC_MIMETYPES, COMIC_BOOK_SUFFIXES, EPUB_MIMETYPE, MOBI_MIMETYPE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DOC_PROCESSING_LIMIT = 10
|
||||
|
||||
|
||||
@register_modeler
|
||||
async def doc_modeler(progress_callback: Callable[[int], None] = None):
|
||||
"""Searches for doc files (epub, mobi, pdf, docx, doc, odt, cbz, cbr) and models them."""
|
||||
total_processed = 0
|
||||
while True:
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup, Doc).filter(
|
||||
or_(
|
||||
*[FileGroup.mimetype == mt for mt in DOC_MIMETYPES],
|
||||
*[FileGroup.primary_path.like(f'%{suffix}') for suffix in COMIC_BOOK_SUFFIXES],
|
||||
),
|
||||
FileGroup.indexed == False,
|
||||
).outerjoin(Doc, Doc.file_group_id == FileGroup.id) \
|
||||
.limit(DOC_PROCESSING_LIMIT)
|
||||
file_groups: List[Tuple[FileGroup, Doc]] = list(file_groups)
|
||||
|
||||
processed = 0
|
||||
for file_group, doc in file_groups:
|
||||
processed += 1
|
||||
try:
|
||||
if PYTEST:
|
||||
session.expire(file_group)
|
||||
|
||||
doc = _model_doc(file_group, session)
|
||||
session.add(doc)
|
||||
file_group.model = Doc.__tablename__
|
||||
file_group.indexed = True
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index doc {file_group}', exc_info=e)
|
||||
file_group.indexed = True # Don't retry on failure.
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
session.commit()
|
||||
|
||||
total_processed += len(file_groups)
|
||||
if progress_callback and len(file_groups) > 0:
|
||||
progress_callback(total_processed)
|
||||
|
||||
if processed < DOC_PROCESSING_LIMIT:
|
||||
break
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
def _model_doc(file_group: FileGroup, session: Session) -> Doc:
|
||||
"""Creates a Doc model from a FileGroup. Extracts metadata and creates collections."""
|
||||
doc = session.query(Doc).filter_by(file_group_id=file_group.id).one_or_none()
|
||||
if not doc:
|
||||
doc = Doc(file_group=file_group)
|
||||
session.add(doc)
|
||||
|
||||
metadata = extract_metadata(file_group)
|
||||
|
||||
if metadata.title:
|
||||
file_group.title = file_group.a_text = metadata.title
|
||||
if metadata.author:
|
||||
file_group.author = file_group.b_text = metadata.author
|
||||
if metadata.text:
|
||||
text = truncate_object_bytes(metadata.text, FILE_MAX_TEXT_SIZE)
|
||||
text = split_lines_by_length(text)
|
||||
file_group.d_text = text
|
||||
|
||||
# Set c_text to filename for search.
|
||||
file_title = Indexer.get_title(file_group.primary_path)
|
||||
file_group.c_text = file_title
|
||||
|
||||
if not file_group.title:
|
||||
file_group.title = file_title
|
||||
|
||||
if metadata.published_date:
|
||||
file_group.published_datetime = metadata.published_date
|
||||
|
||||
# Update Doc-specific fields.
|
||||
doc.publisher = metadata.publisher
|
||||
doc.language = metadata.language
|
||||
doc.page_count = metadata.page_count
|
||||
doc.subject = metadata.subject
|
||||
doc.description = metadata.description
|
||||
doc.size = file_group.primary_path.stat().st_size if file_group.primary_path.exists() else None
|
||||
|
||||
# Handle cover.
|
||||
_handle_doc_cover(doc, file_group, metadata)
|
||||
|
||||
# Store data dict.
|
||||
data = file_group.data or {}
|
||||
if metadata.title:
|
||||
data['title'] = metadata.title
|
||||
if metadata.author:
|
||||
data['author'] = metadata.author
|
||||
# Store primary doc path as relative filename.
|
||||
data['doc_path'] = file_group.primary_path.name
|
||||
file_group.data = data
|
||||
|
||||
# Auto-create collections.
|
||||
_auto_create_collections(session, file_group, metadata)
|
||||
|
||||
doc.flush()
|
||||
return doc
|
||||
|
||||
|
||||
def _handle_doc_cover(doc: Doc, file_group: FileGroup, metadata):
|
||||
"""Handle cover extraction and storage for a doc."""
|
||||
|
||||
# Check for Calibre cover first (epub only) — must happen before poster check
|
||||
# since Calibre cover has a different stem ("cover.jpg" not matching epub stem).
|
||||
if file_group.mimetype and file_group.mimetype.startswith('application/epub'):
|
||||
if calibre_cover := discover_calibre_cover(file_group.primary_path):
|
||||
doc._hide_calibre_files(file_group)
|
||||
file_group.append_files(calibre_cover)
|
||||
file_group.data = file_group.data or {}
|
||||
file_group.data['cover_path'] = calibre_cover.name
|
||||
return
|
||||
|
||||
# Use existing poster files if present.
|
||||
poster_files = file_group.my_poster_files()
|
||||
if poster_files:
|
||||
file_group.data = file_group.data or {}
|
||||
file_group.data['cover_path'] = poster_files[0]['path'].name
|
||||
return
|
||||
|
||||
# Use cover from metadata extraction (epub internal cover, PDF first page, etc.)
|
||||
cover_bytes = metadata.cover_bytes
|
||||
if cover_bytes:
|
||||
cover_path = file_group.primary_path.with_suffix('.jpeg')
|
||||
cover_path.write_bytes(cover_bytes)
|
||||
file_group.append_files(cover_path)
|
||||
file_group.data = file_group.data or {}
|
||||
file_group.data['cover_path'] = cover_path.name
|
||||
|
||||
|
||||
def _auto_create_collections(session: Session, file_group: FileGroup, metadata):
|
||||
"""Auto-create author/subject collections and link the file_group."""
|
||||
if metadata.author:
|
||||
for author in split_authors(metadata.author):
|
||||
author = normalize_author(author)
|
||||
if author and is_valid_author(author):
|
||||
col = get_or_create_author_collection(session, author)
|
||||
if col:
|
||||
col.add_file_group(file_group, session=session)
|
||||
|
||||
if metadata.subject:
|
||||
for subject in split_subjects(metadata.subject):
|
||||
subject = normalize_subject(subject)
|
||||
if subject and is_valid_subject(subject):
|
||||
col = get_or_create_subject_collection(session, subject)
|
||||
if col:
|
||||
col.add_file_group(file_group, session=session)
|
||||
|
||||
|
||||
@register_refresh_cleanup
|
||||
def doc_cleanup(test_directory=None):
|
||||
"""Remove empty author/subject collections after refresh."""
|
||||
from sqlalchemy import func
|
||||
from wrolpi.collections.models import Collection, CollectionItem
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
empty_collections = session.query(Collection).filter(
|
||||
Collection.kind.in_(['author', 'subject']),
|
||||
).outerjoin(CollectionItem).group_by(Collection.id).having(
|
||||
func.count(CollectionItem.id) == 0,
|
||||
).all()
|
||||
for collection in empty_collections:
|
||||
logger.info(f'Removing empty {collection.kind} collection: {collection.name}')
|
||||
session.delete(collection)
|
||||
85
modules/docs/api.py
Normal file
85
modules/docs/api.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
from http import HTTPStatus
|
||||
|
||||
from sanic import response, Request, Blueprint
|
||||
from sanic_ext import validate
|
||||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi.api_utils import json_response
|
||||
from wrolpi.common import logger, wrol_mode_check, api_param_limiter
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.schema import JSONErrorResponse
|
||||
from . import schema
|
||||
from .lib import get_statistics, _doc_response, _get_doc, _get_doc_by_file_group, _delete_docs, _search_docs
|
||||
|
||||
NAME = 'docs'
|
||||
|
||||
docs_bp = Blueprint('Docs', '/api/docs')
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
@docs_bp.get('/statistics')
|
||||
@openapi.response(HTTPStatus.OK, schema.DocStatisticsResponse)
|
||||
async def statistics(_: Request):
|
||||
ret = get_statistics()
|
||||
return json_response(ret, HTTPStatus.OK)
|
||||
|
||||
|
||||
@docs_bp.get('/<doc_id:int>')
|
||||
@openapi.description('Get a doc by doc ID')
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
async def get_doc(request: Request, doc_id: int):
|
||||
session = request.ctx.session
|
||||
doc = _get_doc(session, doc_id)
|
||||
return json_response(_doc_response(doc))
|
||||
|
||||
|
||||
@docs_bp.get('/view/<file_group_id:int>')
|
||||
@openapi.description('Get a doc by file_group ID')
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
async def get_doc_by_file_group(request: Request, file_group_id: int):
|
||||
session = request.ctx.session
|
||||
doc = _get_doc_by_file_group(session, file_group_id)
|
||||
return json_response(_doc_response(doc))
|
||||
|
||||
|
||||
@docs_bp.delete('/<doc_ids:int>', name='doc_delete_one')
|
||||
@docs_bp.delete('/<doc_ids:[0-9,]+>', name='doc_delete_many')
|
||||
@openapi.description('Delete Docs')
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
@wrol_mode_check
|
||||
async def delete_docs(_: Request, doc_ids: str):
|
||||
try:
|
||||
doc_ids = [int(i) for i in str(doc_ids).split(',')]
|
||||
except ValueError:
|
||||
raise ValidationError('Could not parse doc ids')
|
||||
_delete_docs(*doc_ids)
|
||||
return response.empty()
|
||||
|
||||
|
||||
doc_limit_limiter = api_param_limiter(100)
|
||||
|
||||
|
||||
@docs_bp.post('/search')
|
||||
@openapi.definition(
|
||||
summary='Search docs with filtering',
|
||||
body=schema.DocSearchRequest,
|
||||
)
|
||||
@openapi.response(HTTPStatus.OK, schema.DocSearchResponse)
|
||||
@validate(schema.DocSearchRequest)
|
||||
async def search_docs(_: Request, body: schema.DocSearchRequest):
|
||||
limit = doc_limit_limiter(body.limit)
|
||||
offset = body.offset or 0
|
||||
file_groups, total = _search_docs(
|
||||
search_str=body.search_str,
|
||||
author=body.author,
|
||||
subject=body.subject,
|
||||
language=body.language,
|
||||
mimetype=body.mimetype,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
order_by=body.order_by,
|
||||
tag_names=body.tag_names,
|
||||
)
|
||||
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
|
||||
return json_response(ret)
|
||||
26
modules/docs/conftest.py
Normal file
26
modules/docs/conftest.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import pytest
|
||||
|
||||
from modules.docs.models import Doc
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def doc_factory(test_session, test_directory):
|
||||
"""Factory for creating Doc records for testing."""
|
||||
count = 0
|
||||
|
||||
def factory(file_group=None, **kwargs):
|
||||
nonlocal count
|
||||
count += 1
|
||||
if file_group is None:
|
||||
from wrolpi.files.models import FileGroup
|
||||
path = test_directory / f'test_doc_{count}.pdf'
|
||||
path.write_bytes(b'%PDF-1.4 test')
|
||||
file_group = FileGroup.from_paths(test_session, path)
|
||||
test_session.flush()
|
||||
|
||||
doc = Doc(file_group=file_group, **kwargs)
|
||||
test_session.add(doc)
|
||||
test_session.flush()
|
||||
return doc
|
||||
|
||||
return factory
|
||||
428
modules/docs/extractors.py
Normal file
428
modules/docs/extractors.py
Normal file
|
|
@ -0,0 +1,428 @@
|
|||
import dataclasses
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import zipfile
|
||||
|
||||
import pytz
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.common import truncate_generator_bytes, truncate_object_bytes, get_html_soup
|
||||
from wrolpi.files.indexers import Indexer
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import FILE_MAX_PDF_SIZE, FILE_MAX_TEXT_SIZE
|
||||
|
||||
try:
|
||||
import ebooklib
|
||||
from ebooklib import epub
|
||||
except ImportError:
|
||||
ebooklib = None
|
||||
epub = None
|
||||
|
||||
try:
|
||||
import pymupdf
|
||||
except ImportError:
|
||||
pymupdf = None
|
||||
|
||||
try:
|
||||
import docx
|
||||
except ImportError:
|
||||
docx = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocMetadata:
|
||||
title: str = None
|
||||
author: str = None
|
||||
publisher: str = None
|
||||
published_date: datetime.datetime = None
|
||||
page_count: int = None
|
||||
language: str = None
|
||||
subject: str = None
|
||||
description: str = None
|
||||
cover_bytes: bytes = None
|
||||
text: str = None
|
||||
|
||||
|
||||
def extract_metadata(file_group: FileGroup) -> DocMetadata:
|
||||
"""Extract metadata from a FileGroup based on its mimetype."""
|
||||
mimetype = file_group.mimetype or ''
|
||||
path = file_group.primary_path
|
||||
|
||||
if mimetype.startswith('application/epub'):
|
||||
return extract_epub(path)
|
||||
elif mimetype == 'application/x-mobipocket-ebook':
|
||||
# MOBI files can't be easily parsed for metadata.
|
||||
return DocMetadata(title=Indexer.get_title(path))
|
||||
elif mimetype == 'application/pdf':
|
||||
return extract_pdf(path)
|
||||
elif mimetype == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
|
||||
return extract_docx(path)
|
||||
elif mimetype == 'application/msword':
|
||||
return extract_doc(path)
|
||||
elif mimetype == 'application/vnd.oasis.opendocument.text':
|
||||
return extract_odt(path)
|
||||
elif mimetype in ('application/vnd.comicbook+zip', 'application/x-cbz',
|
||||
'application/vnd.comicbook-rar', 'application/x-cbr'):
|
||||
return extract_cbz(path)
|
||||
elif path.suffix.lower() in ('.cbz', '.cbr', '.cbt', '.cb7'):
|
||||
return extract_cbz(path)
|
||||
|
||||
return DocMetadata(title=Indexer.get_title(path))
|
||||
|
||||
|
||||
def _extract_text_from_html(html: str) -> str:
|
||||
soup = get_html_soup(html)
|
||||
return soup.get_text()
|
||||
|
||||
|
||||
def extract_epub(path: pathlib.Path) -> DocMetadata:
|
||||
"""Extract metadata from an EPUB file using ebooklib."""
|
||||
if ebooklib is None or epub is None:
|
||||
raise ValueError('ebooklib is not installed')
|
||||
|
||||
metadata = DocMetadata()
|
||||
|
||||
try:
|
||||
book = epub.read_epub(path, options=dict(ignore_ncx=True))
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to read epub {path}', exc_info=e)
|
||||
metadata.title = Indexer.get_title(path)
|
||||
return metadata
|
||||
|
||||
for key, value in book.metadata.items():
|
||||
if 'title' in value:
|
||||
metadata.title = metadata.title or value['title'][0][0]
|
||||
if 'creator' in value:
|
||||
metadata.author = metadata.author or value['creator'][0][0]
|
||||
if 'publisher' in value:
|
||||
metadata.publisher = metadata.publisher or value['publisher'][0][0]
|
||||
if 'language' in value:
|
||||
metadata.language = metadata.language or value['language'][0][0]
|
||||
if 'description' in value:
|
||||
metadata.description = metadata.description or value['description'][0][0]
|
||||
if 'subject' in value:
|
||||
metadata.subject = metadata.subject or value['subject'][0][0]
|
||||
|
||||
# Extract text.
|
||||
text_parts = []
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||||
text_parts.append(_extract_text_from_html(doc.content))
|
||||
metadata.text = ''.join(text_parts)
|
||||
|
||||
# Extract cover.
|
||||
try:
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_COVER):
|
||||
metadata.cover_bytes = doc.content
|
||||
break
|
||||
if not metadata.cover_bytes:
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_IMAGE):
|
||||
metadata.cover_bytes = doc.content
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to extract cover from {path}', exc_info=e)
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def _parse_pdf_date(date_str: str):
|
||||
"""Parse PDF date format 'D:YYYYMMDDHHmmSS+HH'mm'' to datetime."""
|
||||
if not date_str:
|
||||
return None
|
||||
try:
|
||||
parsed = dates.strpdate(date_str)
|
||||
if parsed and not parsed.tzinfo:
|
||||
parsed = parsed.astimezone(pytz.UTC)
|
||||
return parsed
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def extract_pdf(path: pathlib.Path) -> DocMetadata:
|
||||
"""Extract metadata from a PDF using pymupdf."""
|
||||
if pymupdf is None:
|
||||
logger.error(f'Cannot extract PDF metadata: pymupdf is not installed')
|
||||
return DocMetadata(title=Indexer.get_title(path))
|
||||
|
||||
metadata = DocMetadata()
|
||||
doc = None
|
||||
try:
|
||||
doc = pymupdf.open(path)
|
||||
|
||||
pdf_meta = doc.metadata or {}
|
||||
if (title := pdf_meta.get('title')) and title.lower() != 'unknown':
|
||||
metadata.title = title
|
||||
if (author := pdf_meta.get('author')) and author.lower() != 'unknown':
|
||||
metadata.author = author
|
||||
if subject := pdf_meta.get('subject'):
|
||||
metadata.subject = subject
|
||||
|
||||
metadata.page_count = doc.page_count
|
||||
|
||||
if creation_date := pdf_meta.get('creationDate'):
|
||||
metadata.published_date = _parse_pdf_date(creation_date)
|
||||
|
||||
# Extract text.
|
||||
if path.stat().st_size <= FILE_MAX_PDF_SIZE:
|
||||
try:
|
||||
words = []
|
||||
for page in doc:
|
||||
text = page.get_text()
|
||||
text = text.replace('\x00', '\uFFFD').strip()
|
||||
if text:
|
||||
words.append(text)
|
||||
metadata.text = '\n'.join(
|
||||
truncate_generator_bytes(iter(words), FILE_MAX_TEXT_SIZE))
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract PDF text: {path}', exc_info=e)
|
||||
|
||||
# Extract first page as cover thumbnail.
|
||||
try:
|
||||
if doc.page_count > 0:
|
||||
page = doc[0]
|
||||
pix = page.get_pixmap(matrix=pymupdf.Matrix(0.5, 0.5))
|
||||
metadata.cover_bytes = pix.tobytes('jpeg')
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to render PDF cover: {path}', exc_info=e)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract PDF metadata: {path}', exc_info=e)
|
||||
metadata.title = Indexer.get_title(path)
|
||||
finally:
|
||||
if doc:
|
||||
doc.close()
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def extract_docx(path: pathlib.Path) -> DocMetadata:
|
||||
"""Extract metadata from a DOCX file."""
|
||||
if docx is None:
|
||||
return DocMetadata(title=Indexer.get_title(path))
|
||||
|
||||
metadata = DocMetadata()
|
||||
try:
|
||||
document = docx.Document(str(path))
|
||||
props = document.core_properties
|
||||
|
||||
metadata.title = props.title or None
|
||||
metadata.author = props.author or None
|
||||
metadata.subject = props.subject or None
|
||||
metadata.description = props.comments or None
|
||||
metadata.language = props.language or None
|
||||
|
||||
if props.created:
|
||||
metadata.published_date = props.created
|
||||
|
||||
# Extract text.
|
||||
text_parts = []
|
||||
for para in document.paragraphs:
|
||||
if para.text:
|
||||
text_parts.append(para.text)
|
||||
metadata.text = truncate_object_bytes(' '.join(text_parts), FILE_MAX_TEXT_SIZE)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract docx metadata: {path}', exc_info=e)
|
||||
metadata.title = Indexer.get_title(path)
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def extract_doc(path: pathlib.Path) -> DocMetadata:
|
||||
"""Extract text from old .doc files using catdoc/textutil."""
|
||||
import subprocess
|
||||
from wrolpi.cmd import CATDOC_PATH, TEXTUTIL_PATH
|
||||
|
||||
metadata = DocMetadata()
|
||||
metadata.title = Indexer.get_title(path)
|
||||
|
||||
try:
|
||||
if CATDOC_PATH:
|
||||
cmd = (CATDOC_PATH, str(path.absolute()))
|
||||
proc = subprocess.run(cmd, capture_output=True)
|
||||
metadata.text = proc.stdout.decode()
|
||||
elif TEXTUTIL_PATH:
|
||||
cmd = (TEXTUTIL_PATH, '-stdout', '-cat', 'txt', str(path.absolute()))
|
||||
proc = subprocess.run(cmd, capture_output=True)
|
||||
metadata.text = proc.stdout.decode()
|
||||
|
||||
if metadata.text:
|
||||
metadata.text = truncate_object_bytes(metadata.text, FILE_MAX_TEXT_SIZE)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract doc text: {path}', exc_info=e)
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def extract_odt(path: pathlib.Path) -> DocMetadata:
|
||||
"""Extract metadata from an ODT file by parsing meta.xml inside the zip."""
|
||||
metadata = DocMetadata()
|
||||
metadata.title = Indexer.get_title(path)
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(path, 'r') as zf:
|
||||
if 'meta.xml' in zf.namelist():
|
||||
meta_xml = zf.read('meta.xml').decode('utf-8')
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.fromstring(meta_xml)
|
||||
ns = {
|
||||
'office': 'urn:oasis:names:tc:opendocument:xmlns:office:1.0',
|
||||
'dc': 'http://purl.org/dc/elements/1.1/',
|
||||
'meta': 'urn:oasis:names:tc:opendocument:xmlns:meta:1.0',
|
||||
}
|
||||
office_meta = root.find('.//office:meta', ns)
|
||||
if office_meta is not None:
|
||||
title_el = office_meta.find('dc:title', ns)
|
||||
if title_el is not None and title_el.text:
|
||||
metadata.title = title_el.text
|
||||
creator_el = office_meta.find('dc:creator', ns)
|
||||
if creator_el is not None and creator_el.text:
|
||||
metadata.author = creator_el.text
|
||||
subject_el = office_meta.find('dc:subject', ns)
|
||||
if subject_el is not None and subject_el.text:
|
||||
metadata.subject = subject_el.text
|
||||
desc_el = office_meta.find('dc:description', ns)
|
||||
if desc_el is not None and desc_el.text:
|
||||
metadata.description = desc_el.text
|
||||
lang_el = office_meta.find('dc:language', ns)
|
||||
if lang_el is not None and lang_el.text:
|
||||
metadata.language = lang_el.text
|
||||
|
||||
# Extract text from content.xml.
|
||||
if 'content.xml' in zf.namelist():
|
||||
content_xml = zf.read('content.xml').decode('utf-8')
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.fromstring(content_xml)
|
||||
# Get all text content.
|
||||
texts = []
|
||||
for elem in root.iter():
|
||||
if elem.text:
|
||||
texts.append(elem.text)
|
||||
if elem.tail:
|
||||
texts.append(elem.tail)
|
||||
metadata.text = truncate_object_bytes(' '.join(texts), FILE_MAX_TEXT_SIZE)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract ODT metadata: {path}', exc_info=e)
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def _list_comic_archive(path: pathlib.Path):
|
||||
"""Return (file_names, read_func) for a comic book archive.
|
||||
|
||||
read_func(name) returns the bytes of the named member."""
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in ('.cbz', '.zip'):
|
||||
zf = zipfile.ZipFile(path, 'r')
|
||||
return zf.namelist(), zf.read, zf
|
||||
elif suffix in ('.cbt',):
|
||||
import tarfile as _tarfile
|
||||
tf = _tarfile.open(path, 'r:*')
|
||||
names = [m.name for m in tf.getmembers() if not m.isdir()]
|
||||
|
||||
def read_tar(name):
|
||||
f = tf.extractfile(name)
|
||||
return f.read() if f else b''
|
||||
|
||||
return names, read_tar, tf
|
||||
elif suffix in ('.cbr',):
|
||||
import rarfile
|
||||
rf = rarfile.RarFile(path, 'r')
|
||||
return rf.namelist(), rf.read, rf
|
||||
elif suffix in ('.cb7',):
|
||||
import py7zr
|
||||
import tempfile
|
||||
sz = py7zr.SevenZipFile(path, 'r')
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
sz.extractall(tmpdir)
|
||||
sz.close()
|
||||
names = []
|
||||
for root, dirs, files in os.walk(tmpdir):
|
||||
for f in files:
|
||||
rel = os.path.relpath(os.path.join(root, f), tmpdir)
|
||||
names.append(rel)
|
||||
|
||||
def read_7z(name):
|
||||
return (pathlib.Path(tmpdir) / name).read_bytes()
|
||||
|
||||
class Cleanup:
|
||||
def close(self):
|
||||
import shutil
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
return names, read_7z, Cleanup()
|
||||
return [], lambda n: b'', None
|
||||
|
||||
|
||||
def extract_cbz(path: pathlib.Path) -> DocMetadata:
|
||||
"""Extract metadata from comic book archives (CBZ, CBT, CBR, CB7)."""
|
||||
metadata = DocMetadata()
|
||||
metadata.title = Indexer.get_title(path)
|
||||
|
||||
try:
|
||||
names, read_func, ctx = _list_comic_archive(path)
|
||||
try:
|
||||
# Look for ComicInfo.xml.
|
||||
for name in names:
|
||||
if name.lower().endswith('comicinfo.xml'):
|
||||
try:
|
||||
comic_xml = read_func(name).decode('utf-8')
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.fromstring(comic_xml)
|
||||
title_el = root.find('Title')
|
||||
series_el = root.find('Series')
|
||||
number_el = root.find('Number')
|
||||
if series_el is not None and series_el.text:
|
||||
# Use "Series #Number" as title if available.
|
||||
title = series_el.text
|
||||
if number_el is not None and number_el.text:
|
||||
title = f'{title} #{number_el.text}'
|
||||
metadata.title = title
|
||||
elif title_el is not None and title_el.text:
|
||||
metadata.title = title_el.text
|
||||
writer_el = root.find('Writer')
|
||||
if writer_el is not None and writer_el.text:
|
||||
metadata.author = writer_el.text
|
||||
summary_el = root.find('Summary')
|
||||
if summary_el is not None and summary_el.text:
|
||||
metadata.description = summary_el.text
|
||||
page_count_el = root.find('PageCount')
|
||||
if page_count_el is not None and page_count_el.text:
|
||||
metadata.page_count = int(page_count_el.text)
|
||||
publisher_el = root.find('Publisher')
|
||||
if publisher_el is not None and publisher_el.text:
|
||||
metadata.publisher = publisher_el.text
|
||||
lang_el = root.find('LanguageISO')
|
||||
if lang_el is not None and lang_el.text:
|
||||
metadata.language = lang_el.text
|
||||
genre_el = root.find('Genre')
|
||||
if genre_el is not None and genre_el.text:
|
||||
metadata.subject = genre_el.text
|
||||
# Build published_date from Year/Month/Day.
|
||||
year_el = root.find('Year')
|
||||
if year_el is not None and year_el.text:
|
||||
year = int(year_el.text)
|
||||
month_el = root.find('Month')
|
||||
day_el = root.find('Day')
|
||||
month = int(month_el.text) if month_el is not None and month_el.text else 1
|
||||
day = int(day_el.text) if day_el is not None and day_el.text else 1
|
||||
metadata.published_date = datetime.datetime(year, month, day, tzinfo=pytz.UTC)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to parse ComicInfo.xml: {path}', exc_info=e)
|
||||
break
|
||||
|
||||
# Extract first image as cover.
|
||||
image_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.webp')
|
||||
image_names = sorted([n for n in names
|
||||
if any(n.lower().endswith(ext) for ext in image_extensions)])
|
||||
if image_names:
|
||||
metadata.cover_bytes = read_func(image_names[0])
|
||||
finally:
|
||||
if ctx:
|
||||
ctx.close()
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract comic metadata: {path}', exc_info=e)
|
||||
|
||||
return metadata
|
||||
417
modules/docs/lib.py
Normal file
417
modules/docs/lib.py
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
import html
|
||||
import logging
|
||||
import pathlib
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import asc, desc, func, nullslast
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.collections.types import collection_type_registry
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Register author and subject collection types.
|
||||
collection_type_registry.register(
|
||||
'author',
|
||||
lambda n: bool(n and n.strip()),
|
||||
'Non-empty author name'
|
||||
)
|
||||
|
||||
collection_type_registry.register(
|
||||
'subject',
|
||||
lambda n: bool(n and n.strip()),
|
||||
'Non-empty subject name'
|
||||
)
|
||||
|
||||
# Junk author patterns to reject.
|
||||
JUNK_AUTHORS = {
|
||||
'administrator', 'admin', 'unknown', 'preferred customer', 'user',
|
||||
'owner', 'default', 'none', 'n/a', 'na', 'test', 'root',
|
||||
}
|
||||
|
||||
JUNK_SUBJECTS = {
|
||||
'unknown', 'n/a', 'na', 'none', 'test', 'default',
|
||||
}
|
||||
|
||||
# Regex for URL-like strings.
|
||||
URL_PATTERN = re.compile(r'https?://|www\.|\.com|\.org|\.net')
|
||||
|
||||
# Matches year ranges like 1483-1546, -1938, 1564-1616.
|
||||
YEAR_RANGE_PATTERN = re.compile(r'^-?\d{0,4}-?\d{4}$')
|
||||
|
||||
# Matches bracket suffixes like [from old catalog].
|
||||
BRACKET_SUFFIX_PATTERN = re.compile(r'\s*\]?\s*\[.*$')
|
||||
|
||||
# Matches trailing birth-death years like 1843-1920 at end of string.
|
||||
TRAILING_YEARS_PATTERN = re.compile(r'\s+\d{4}-\d{4}$')
|
||||
|
||||
# Matches pure numeric/code patterns: 00110001, 062447, 105642-01, 01/2004.
|
||||
NUMERIC_CODE_PATTERN = re.compile(r'^\d[\d\-/]*$')
|
||||
|
||||
# Matches date strings like "14 June 2006".
|
||||
DATE_STRING_PATTERN = re.compile(r'^\d{1,2}\s+\w+\s+\d{4}$')
|
||||
|
||||
# Matches HTML entities like • or &
|
||||
HTML_ENTITY_PATTERN = re.compile(r'&\w+;')
|
||||
|
||||
# Matches initials pattern like J.K. (used to avoid stripping trailing period).
|
||||
INITIALS_PATTERN = re.compile(r'\.\w\.')
|
||||
|
||||
|
||||
def normalize_author(author: str) -> Optional[str]:
|
||||
"""Clean an author string before validation."""
|
||||
if not author or not author.strip():
|
||||
return None
|
||||
author = author.strip()
|
||||
|
||||
# Remove bracket suffixes like [from old catalog].
|
||||
author = BRACKET_SUFFIX_PATTERN.sub('', author)
|
||||
|
||||
# Strip surrounding quotes.
|
||||
if len(author) >= 2 and author[0] == '"' and author[-1] == '"':
|
||||
author = author[1:-1]
|
||||
# Also strip leading quote left after bracket removal.
|
||||
if author.startswith('"'):
|
||||
author = author[1:]
|
||||
|
||||
# Remove surrounding parentheses.
|
||||
if len(author) >= 2 and author[0] == '(' and author[-1] == ')':
|
||||
author = author[1:-1]
|
||||
|
||||
# Remove trailing birth-death years.
|
||||
author = TRAILING_YEARS_PATTERN.sub('', author)
|
||||
|
||||
# Strip trailing period, but not if it's part of initials (e.g. J.K.).
|
||||
if author.endswith('.') and not INITIALS_PATTERN.search(author):
|
||||
author = author[:-1]
|
||||
|
||||
# Strip trailing comma.
|
||||
author = author.rstrip(',')
|
||||
|
||||
# Title-case for consistent casing.
|
||||
author = author.title()
|
||||
|
||||
author = author.strip()
|
||||
return author if author else None
|
||||
|
||||
|
||||
def normalize_subject(subject: str) -> Optional[str]:
|
||||
"""Clean a subject string before validation."""
|
||||
if not subject or not subject.strip():
|
||||
return None
|
||||
subject = subject.strip()
|
||||
|
||||
# Decode HTML entities.
|
||||
subject = html.unescape(subject)
|
||||
|
||||
# Title-case for consistent casing.
|
||||
subject = subject.title()
|
||||
|
||||
# Truncate to 150 chars.
|
||||
if len(subject) > 150:
|
||||
subject = subject[:150]
|
||||
|
||||
subject = subject.strip()
|
||||
return subject if subject else None
|
||||
|
||||
|
||||
# Matches pure digits.
|
||||
PURE_DIGITS_PATTERN = re.compile(r'^\d+$')
|
||||
|
||||
# Matches any letter character.
|
||||
HAS_LETTER_PATTERN = re.compile(r'[a-zA-Z]')
|
||||
|
||||
|
||||
def is_valid_author(author: str) -> bool:
|
||||
"""Check if an author string looks like a real author name."""
|
||||
if not author or not author.strip():
|
||||
return False
|
||||
author = author.strip()
|
||||
if len(author) > 100:
|
||||
return False
|
||||
if len(author) < 2:
|
||||
return False
|
||||
if author.lower() in JUNK_AUTHORS:
|
||||
return False
|
||||
if URL_PATTERN.search(author):
|
||||
return False
|
||||
# Reject pure year ranges and numbers.
|
||||
if YEAR_RANGE_PATTERN.match(author):
|
||||
return False
|
||||
if PURE_DIGITS_PATTERN.match(author):
|
||||
return False
|
||||
# Reject strings with no letter characters.
|
||||
if not HAS_LETTER_PATTERN.search(author):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_subject(subject: str) -> bool:
|
||||
"""Check if a subject string looks like a real subject."""
|
||||
if not subject or not subject.strip():
|
||||
return False
|
||||
subject = subject.strip()
|
||||
if len(subject) < 2:
|
||||
return False
|
||||
if len(subject) > 150:
|
||||
return False
|
||||
if subject.lower() in JUNK_SUBJECTS:
|
||||
return False
|
||||
if URL_PATTERN.search(subject):
|
||||
return False
|
||||
# Reject pure numeric/code patterns.
|
||||
if NUMERIC_CODE_PATTERN.match(subject):
|
||||
return False
|
||||
# Reject date strings.
|
||||
if DATE_STRING_PATTERN.match(subject):
|
||||
return False
|
||||
# Reject strings with HTML entities.
|
||||
if HTML_ENTITY_PATTERN.search(subject):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# Splits multiple authors on common separators.
|
||||
AUTHOR_SPLIT_PATTERN = re.compile(r'[;&]|,\s*(?:and|&)\s*|,\s+')
|
||||
|
||||
|
||||
def split_authors(author_str: str) -> List[str]:
|
||||
"""Split an author string that may contain multiple authors."""
|
||||
if not author_str:
|
||||
return []
|
||||
# Split on common separators.
|
||||
authors = AUTHOR_SPLIT_PATTERN.split(author_str)
|
||||
return [a.strip() for a in authors if a.strip()]
|
||||
|
||||
|
||||
# Splits multiple subjects on semicolons and bare commas.
|
||||
SUBJECT_SPLIT_PATTERN = re.compile(r';|,(?!\s)')
|
||||
|
||||
|
||||
def split_subjects(subject_str: str) -> List[str]:
|
||||
"""Split a subject string that may contain multiple subjects."""
|
||||
if not subject_str:
|
||||
return []
|
||||
# Split on semicolons and bare commas (comma NOT followed by a space).
|
||||
subjects = SUBJECT_SPLIT_PATTERN.split(subject_str)
|
||||
return [s.strip() for s in subjects if s.strip()]
|
||||
|
||||
|
||||
def get_or_create_author_collection(session: Session, author_name: str) -> Optional[Collection]:
|
||||
"""Find or create a Collection for an author."""
|
||||
if not author_name or not author_name.strip():
|
||||
return None
|
||||
|
||||
author_name = author_name.strip()
|
||||
collection = session.query(Collection).filter(
|
||||
func.lower(Collection.name) == author_name.lower(),
|
||||
Collection.kind == 'author',
|
||||
).first()
|
||||
if not collection:
|
||||
collection = Collection(name=author_name, kind='author')
|
||||
session.add(collection)
|
||||
session.flush()
|
||||
logger.debug(f'Created author collection: {author_name}')
|
||||
return collection
|
||||
|
||||
|
||||
def get_or_create_subject_collection(session: Session, subject_name: str) -> Optional[Collection]:
|
||||
"""Find or create a Collection for a subject."""
|
||||
if not subject_name or not subject_name.strip():
|
||||
return None
|
||||
|
||||
subject_name = subject_name.strip()
|
||||
collection = session.query(Collection).filter(
|
||||
func.lower(Collection.name) == subject_name.lower(),
|
||||
Collection.kind == 'subject',
|
||||
).first()
|
||||
if not collection:
|
||||
collection = Collection(name=subject_name, kind='subject')
|
||||
session.add(collection)
|
||||
session.flush()
|
||||
logger.debug(f'Created subject collection: {subject_name}')
|
||||
return collection
|
||||
|
||||
|
||||
async def search_authors_by_name(session: Session, name: str, limit: int = 5) -> List[dict]:
|
||||
"""Search for author collections by partial name."""
|
||||
collections = session.query(Collection) \
|
||||
.filter(Collection.kind == 'author') \
|
||||
.filter(Collection.name.ilike(f'%{name}%')) \
|
||||
.order_by(asc(Collection.name)) \
|
||||
.limit(limit) \
|
||||
.all()
|
||||
return [{'id': c.id, 'name': c.name} for c in collections]
|
||||
|
||||
|
||||
async def search_subjects_by_name(session: Session, name: str, limit: int = 5) -> List[dict]:
|
||||
"""Search for subject collections by partial name."""
|
||||
collections = session.query(Collection) \
|
||||
.filter(Collection.kind == 'subject') \
|
||||
.filter(Collection.name.ilike(f'%{name}%')) \
|
||||
.order_by(asc(Collection.name)) \
|
||||
.limit(limit) \
|
||||
.all()
|
||||
return [{'id': c.id, 'name': c.name} for c in collections]
|
||||
|
||||
|
||||
def discover_calibre_cover(ebook_path: pathlib.Path):
|
||||
"""Calibre puts a cover file in a directory which contains only one ebook (but multiple formats of the ebook),
|
||||
this function will return the path of the cover if the ebook_path is in such a situation."""
|
||||
if not ebook_path.is_file():
|
||||
raise ValueError(f'Invalid ebook path {ebook_path}')
|
||||
|
||||
paths = list(ebook_path.parent.iterdir())
|
||||
|
||||
ebook_path_stem, _ = split_path_stem_and_suffix(ebook_path)
|
||||
|
||||
cover = None
|
||||
for path in paths:
|
||||
stem, suffix = split_path_stem_and_suffix(path)
|
||||
if path.is_dir():
|
||||
# We don't care about sub directories.
|
||||
continue
|
||||
if stem == ebook_path_stem:
|
||||
# This is one of the formats of the ebook.
|
||||
continue
|
||||
if path.name == 'metadata.opf':
|
||||
# Metadata, this is expected in a Calibre book directory.
|
||||
continue
|
||||
if stem == 'cover' and get_mimetype(path).startswith('image/'):
|
||||
# Discovered the cover.
|
||||
cover = path
|
||||
continue
|
||||
# Some file that does not match the ebook stem, metadata, or cover. This must not be a Calibre book directory.
|
||||
return None
|
||||
return cover
|
||||
|
||||
|
||||
def get_statistics() -> dict:
|
||||
"""Get doc statistics."""
|
||||
from .models import Doc
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
with get_db_session() as session:
|
||||
total = session.query(func.count(Doc.id)).scalar() or 0
|
||||
total_size = session.query(func.sum(Doc.size)).scalar() or 0
|
||||
|
||||
epub_count = session.query(func.count(Doc.id)).join(FileGroup).filter(
|
||||
FileGroup.mimetype.startswith('application/epub')).scalar() or 0
|
||||
pdf_count = session.query(func.count(Doc.id)).join(FileGroup).filter(
|
||||
FileGroup.mimetype == 'application/pdf').scalar() or 0
|
||||
|
||||
author_count = session.query(func.count(Collection.id)).filter(
|
||||
Collection.kind == 'author').scalar() or 0
|
||||
subject_count = session.query(func.count(Collection.id)).filter(
|
||||
Collection.kind == 'subject').scalar() or 0
|
||||
|
||||
return dict(statistics=dict(
|
||||
doc_count=total,
|
||||
epub_count=epub_count,
|
||||
pdf_count=pdf_count,
|
||||
other_count=total - epub_count - pdf_count,
|
||||
total_size=total_size,
|
||||
author_count=author_count,
|
||||
subject_count=subject_count,
|
||||
))
|
||||
|
||||
|
||||
def _doc_response(doc) -> dict:
|
||||
"""Build a JSON response dict for a Doc."""
|
||||
file_group = doc.file_group.__json__()
|
||||
return {
|
||||
'file_group': file_group,
|
||||
'doc': {
|
||||
'id': doc.id,
|
||||
'publisher': doc.publisher,
|
||||
'language': doc.language,
|
||||
'page_count': doc.page_count,
|
||||
'subject': doc.subject,
|
||||
'description': doc.description,
|
||||
'size': doc.size,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _get_doc(session, doc_id: int):
|
||||
from .models import Doc
|
||||
doc = session.query(Doc).filter_by(id=doc_id).one_or_none()
|
||||
if not doc:
|
||||
raise ValidationError(f'Doc {doc_id} not found')
|
||||
return doc
|
||||
|
||||
|
||||
def _get_doc_by_file_group(session, file_group_id: int):
|
||||
from .models import Doc
|
||||
doc = session.query(Doc).filter_by(file_group_id=file_group_id).one_or_none()
|
||||
if not doc:
|
||||
raise ValidationError(f'Doc with file_group_id {file_group_id} not found')
|
||||
return doc
|
||||
|
||||
|
||||
def _delete_docs(*doc_ids):
|
||||
from .models import Doc
|
||||
with get_db_session(commit=True) as session:
|
||||
for doc_id in doc_ids:
|
||||
doc = session.query(Doc).filter_by(id=doc_id).one_or_none()
|
||||
if doc:
|
||||
session.delete(doc)
|
||||
|
||||
|
||||
def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
|
||||
limit=20, offset=0, order_by='published_datetime', tag_names=None):
|
||||
from .models import Doc
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
with get_db_session() as session:
|
||||
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
|
||||
|
||||
if search_str:
|
||||
query = query.filter(FileGroup.textsearch.match(search_str))
|
||||
|
||||
if author:
|
||||
query = query.filter(FileGroup.b_text.ilike(f'%{author}%'))
|
||||
|
||||
if subject:
|
||||
query = query.filter(Doc.subject.ilike(f'%{subject}%'))
|
||||
|
||||
if language:
|
||||
query = query.filter(Doc.language == language)
|
||||
|
||||
if mimetype:
|
||||
query = query.filter(FileGroup.mimetype.startswith(mimetype))
|
||||
|
||||
if tag_names:
|
||||
from wrolpi.tags import Tag, TagFile
|
||||
tagged_fg_ids = session.query(TagFile.file_group_id) \
|
||||
.join(Tag, Tag.id == TagFile.tag_id) \
|
||||
.filter(Tag.name.in_(tag_names)) \
|
||||
.subquery()
|
||||
query = query.filter(FileGroup.id.in_(tagged_fg_ids))
|
||||
|
||||
total = query.count()
|
||||
|
||||
# Ordering.
|
||||
if order_by == 'rank' and search_str:
|
||||
query = query.order_by(
|
||||
desc(func.ts_rank(FileGroup.textsearch, func.websearch_to_tsquery(search_str))),
|
||||
desc(FileGroup.id),
|
||||
)
|
||||
elif order_by == 'published_datetime':
|
||||
query = query.order_by(nullslast(desc(FileGroup.published_datetime)), desc(FileGroup.id))
|
||||
elif order_by == 'size':
|
||||
query = query.order_by(nullslast(desc(Doc.size)))
|
||||
elif order_by == 'title':
|
||||
query = query.order_by(nullslast(asc(FileGroup.a_text)))
|
||||
else:
|
||||
query = query.order_by(desc(FileGroup.id))
|
||||
|
||||
file_groups = query.offset(offset).limit(limit).all()
|
||||
results = [fg.__json__() for fg in file_groups]
|
||||
|
||||
return results, total
|
||||
122
modules/docs/models.py
Normal file
122
modules/docs/models.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import pathlib
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
|
||||
from wrolpi.common import ModelHelper, Base
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
EPUB_MIMETYPE = 'application/epub'
|
||||
MOBI_MIMETYPE = 'application/x-mobipocket-ebook'
|
||||
|
||||
DOC_MIMETYPES = (
|
||||
'application/epub+zip',
|
||||
'application/x-mobipocket-ebook',
|
||||
'application/pdf',
|
||||
'application/msword',
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'application/vnd.oasis.opendocument.text',
|
||||
'application/vnd.comicbook-rar',
|
||||
'application/vnd.comicbook+zip',
|
||||
'application/x-cbz',
|
||||
'application/x-cbr',
|
||||
'application/x-cbt',
|
||||
'application/x-cb7',
|
||||
)
|
||||
|
||||
# Mimetypes that are considered ebook formats.
|
||||
EBOOK_MIMETYPES = (
|
||||
EPUB_MIMETYPE,
|
||||
MOBI_MIMETYPE,
|
||||
)
|
||||
|
||||
|
||||
COMIC_BOOK_SUFFIXES = ('.cbz', '.cbr', '.cbt', '.cb7')
|
||||
|
||||
|
||||
def mimetype_is_doc(mimetype: str) -> bool:
|
||||
"""Check if a mimetype is a doc type that the Doc model handles."""
|
||||
return any(mimetype.startswith(mt) for mt in DOC_MIMETYPES)
|
||||
|
||||
|
||||
class Doc(ModelHelper, Base):
|
||||
__tablename__ = 'doc'
|
||||
__table_args__ = (
|
||||
Index('doc_size_idx', 'size'),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
size = Column(BigInteger)
|
||||
publisher = Column(String)
|
||||
language = Column(String)
|
||||
page_count = Column(Integer)
|
||||
subject = Column(String)
|
||||
description = Column(Text)
|
||||
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
def __repr__(self):
|
||||
path = str(self.file_group.primary_path) if self.file_group else 'None'
|
||||
return f'<Doc id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
|
||||
|
||||
def __json__(self) -> dict:
|
||||
d = self.file_group.__json__()
|
||||
if d.get('data') and d['data'].get('doc_path'):
|
||||
d['data']['doc_path'] = self.file_group.resolve_path(d['data']['doc_path'])
|
||||
if d.get('data') and d['data'].get('cover_path'):
|
||||
d['data']['cover_path'] = self.file_group.resolve_path(d['data']['cover_path'])
|
||||
# Also handle legacy ebook_path key.
|
||||
if d.get('data') and d['data'].get('ebook_path'):
|
||||
d['data']['ebook_path'] = self.file_group.resolve_path(d['data']['ebook_path'])
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def can_model(file_group: FileGroup) -> bool:
|
||||
if file_group.mimetype and mimetype_is_doc(file_group.mimetype):
|
||||
return True
|
||||
if file_group.primary_path and file_group.primary_path.suffix.lower() in COMIC_BOOK_SUFFIXES:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def do_model(session: Session, file_group: FileGroup) -> 'Doc':
|
||||
from modules.docs import _model_doc
|
||||
doc = _model_doc(file_group, session)
|
||||
file_group.model = Doc.__tablename__
|
||||
file_group.indexed = True
|
||||
return doc
|
||||
|
||||
@property
|
||||
def cover_file(self) -> Optional[dict]:
|
||||
posters = self.file_group.my_poster_files()
|
||||
if posters:
|
||||
return posters[0]
|
||||
return None
|
||||
|
||||
@property
|
||||
def cover_path(self) -> Optional[pathlib.Path]:
|
||||
cover = self.cover_file
|
||||
if cover:
|
||||
return cover['path']
|
||||
return None
|
||||
|
||||
def _hide_calibre_files(self, file_group: FileGroup):
|
||||
"""Delete FileGroups of the cover/metadata when this is a Calibre ebook directory."""
|
||||
from modules.docs.lib import discover_calibre_cover
|
||||
path = file_group.primary_path
|
||||
if not discover_calibre_cover(path):
|
||||
raise ValueError(f'Refusing to hide Calibre files when we do not have a Calibre ebook directory!')
|
||||
|
||||
metadata = path.parent / 'metadata.opf'
|
||||
cover = path.parent / 'cover.jpg'
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup).filter(or_(
|
||||
FileGroup.primary_path == str(metadata),
|
||||
FileGroup.primary_path == str(cover),
|
||||
))
|
||||
for fg in file_groups:
|
||||
session.delete(fg)
|
||||
43
modules/docs/schema.py
Normal file
43
modules/docs/schema.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
import dataclasses
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocSearchRequest:
|
||||
search_str: Optional[str] = None
|
||||
author: Optional[str] = None
|
||||
subject: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
mimetype: Optional[str] = None
|
||||
tag_names: Optional[list] = dataclasses.field(default_factory=list)
|
||||
order_by: Optional[str] = 'published_datetime'
|
||||
limit: int = 20
|
||||
offset: int = 0
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocSearchResponse:
|
||||
file_groups: list = dataclasses.field(default_factory=list)
|
||||
totals: dict = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocStatistics:
|
||||
doc_count: int = 0
|
||||
epub_count: int = 0
|
||||
pdf_count: int = 0
|
||||
other_count: int = 0
|
||||
total_size: int = 0
|
||||
author_count: int = 0
|
||||
subject_count: int = 0
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocStatisticsResponse:
|
||||
statistics: dict = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocFileFormatRequest:
|
||||
name: str = ''
|
||||
model_tag: str = 'docs'
|
||||
0
modules/docs/test/__init__.py
Normal file
0
modules/docs/test/__init__.py
Normal file
79
modules/docs/test/test_api.py
Normal file
79
modules/docs/test/test_api.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
"""Tests for the docs API."""
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.docs.models import Doc
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_docs_statistics(async_client, test_session, test_directory, example_epub, refresh_files):
|
||||
"""Doc statistics endpoint returns correct counts."""
|
||||
await refresh_files()
|
||||
|
||||
request, response = await async_client.get('/api/docs/statistics')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
stats = response.json['statistics']
|
||||
assert stats['doc_count'] >= 1
|
||||
assert stats['epub_count'] >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_docs_search(async_client, test_session, test_directory, example_epub, refresh_files):
|
||||
"""Docs search endpoint works."""
|
||||
await refresh_files()
|
||||
|
||||
content = dict(search_str='WROLPi')
|
||||
request, response = await async_client.post('/api/docs/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['file_groups']) >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_doc(async_client, test_session, test_directory, example_epub, refresh_files):
|
||||
"""Single doc endpoint returns the doc."""
|
||||
await refresh_files()
|
||||
|
||||
doc = test_session.query(Doc).first()
|
||||
assert doc
|
||||
|
||||
request, response = await async_client.get(f'/api/docs/{doc.id}')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file_group']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_docs_search_by_tag(async_client, test_session, doc_factory, tag_factory):
|
||||
"""Docs can be searched by tag."""
|
||||
tag = await tag_factory()
|
||||
doc = doc_factory()
|
||||
doc.file_group.add_tag(test_session, tag.name)
|
||||
test_session.commit()
|
||||
|
||||
# Search with matching tag returns the doc.
|
||||
content = dict(tag_names=[tag.name])
|
||||
request, response = await async_client.post('/api/docs/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['totals']['file_groups'] == 1
|
||||
|
||||
# Search with non-existent tag returns nothing.
|
||||
content = dict(tag_names=['nonexistent'])
|
||||
request, response = await async_client.post('/api/docs/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['totals']['file_groups'] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_doc(async_client, test_session, test_directory, example_epub, refresh_files):
|
||||
"""Docs can be deleted."""
|
||||
await refresh_files()
|
||||
|
||||
doc = test_session.query(Doc).first()
|
||||
assert doc
|
||||
doc_id = doc.id
|
||||
|
||||
request, response = await async_client.delete(f'/api/docs/{doc_id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
assert test_session.query(Doc).filter_by(id=doc_id).one_or_none() is None
|
||||
277
modules/docs/test/test_doc_modeler.py
Normal file
277
modules/docs/test/test_doc_modeler.py
Normal file
|
|
@ -0,0 +1,277 @@
|
|||
"""Tests for the doc_modeler."""
|
||||
import shutil
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from modules.docs import doc_modeler
|
||||
from modules.docs.models import Doc
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_epub(async_client, test_session, test_directory, example_epub, example_mobi, refresh_files):
|
||||
"""Doc modeler indexes epub files."""
|
||||
await refresh_files()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
assert doc.file_group.title == 'WROLPi Test Book'
|
||||
assert doc.file_group.mimetype.startswith('application/epub')
|
||||
assert doc.file_group.author == 'roland'
|
||||
assert doc.file_group.a_text, 'Book title was not indexed'
|
||||
assert doc.file_group.b_text, 'Book author was not indexed'
|
||||
assert doc.file_group.d_text, 'Book text was not indexed'
|
||||
assert doc.file_group.model == 'doc'
|
||||
|
||||
# Both ebooks are assumed to be the same book.
|
||||
assert (epubs := doc.file_group.my_files('application/epub+zip')) and len(epubs) == 1
|
||||
assert (mobis := doc.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1
|
||||
# Cover was discovered.
|
||||
assert len(doc.file_group.my_poster_files()) == 1
|
||||
|
||||
# A second refresh should not re-model the already-indexed doc.
|
||||
with patch('modules.docs._model_doc', side_effect=AssertionError('_model_doc should not be called')):
|
||||
await refresh_files()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_pdf(async_client, test_session, test_directory, example_pdf, refresh_files):
|
||||
"""Doc modeler indexes PDF files."""
|
||||
await refresh_files()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
assert doc.file_group.mimetype == 'application/pdf'
|
||||
assert doc.file_group.model == 'doc'
|
||||
assert doc.file_group.indexed is True
|
||||
assert doc.size # Has a size.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_do_model_sets_model(test_session, example_pdf):
|
||||
"""Doc.do_model sets file_group.model so the FileGroup is recognized as a doc."""
|
||||
files_lib._upsert_files([example_pdf], now())
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
# Reset to simulate a newly discovered file.
|
||||
file_group.model = None
|
||||
file_group.indexed = False
|
||||
test_session.commit()
|
||||
|
||||
file_group.do_model(test_session)
|
||||
test_session.commit()
|
||||
|
||||
assert file_group.model == 'doc', 'do_model must set file_group.model'
|
||||
assert file_group.indexed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory):
|
||||
"""Doc modeler processes more than one batch of files (limit=10)."""
|
||||
ebook_dir = test_directory / 'ebooks'
|
||||
ebook_dir.mkdir(parents=True)
|
||||
|
||||
num_docs = 15
|
||||
for i in range(num_docs):
|
||||
path = ebook_dir / f'test_ebook_{i:03d}.epub'
|
||||
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', path)
|
||||
|
||||
# Create FileGroups.
|
||||
for path in ebook_dir.iterdir():
|
||||
fg = FileGroup.from_paths(test_session, path)
|
||||
assert fg.mimetype.startswith('application/epub')
|
||||
test_session.commit()
|
||||
|
||||
await doc_modeler()
|
||||
|
||||
doc_count = test_session.query(Doc).count()
|
||||
assert doc_count == num_docs, \
|
||||
f'doc_modeler should process ALL {num_docs} files, but only processed {doc_count}'
|
||||
|
||||
still_unindexed = test_session.query(FileGroup).filter(
|
||||
FileGroup.indexed == False,
|
||||
FileGroup.mimetype == 'application/epub+zip',
|
||||
).count()
|
||||
assert still_unindexed == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_reindex_creates_collections(async_client, test_session, test_directory, example_epub,
|
||||
refresh_files):
|
||||
"""When a doc is re-indexed (indexed=false), author/subject collections should be re-created."""
|
||||
from wrolpi.collections.models import Collection
|
||||
|
||||
from wrolpi.db import get_db_session
|
||||
|
||||
await refresh_files()
|
||||
|
||||
# Author collection was created during first modeling.
|
||||
with get_db_session() as session:
|
||||
doc = session.query(Doc).one()
|
||||
assert doc.file_group.author == 'roland'
|
||||
author_col = session.query(Collection).filter_by(kind='author', name='Roland').one_or_none()
|
||||
assert author_col, 'Author collection should exist after first indexing'
|
||||
|
||||
# Simulate post-migration state: delete the collection and mark as unindexed.
|
||||
with get_db_session(commit=True) as session:
|
||||
author_col = session.query(Collection).filter_by(kind='author', name='Roland').one()
|
||||
session.delete(author_col)
|
||||
doc = session.query(Doc).one()
|
||||
doc.file_group.indexed = False
|
||||
|
||||
# Verify collection is gone.
|
||||
with get_db_session() as session:
|
||||
assert session.query(Collection).filter_by(kind='author', name='Roland').one_or_none() is None
|
||||
|
||||
# Re-run the modeler.
|
||||
await doc_modeler()
|
||||
|
||||
# Author collection should be re-created.
|
||||
with get_db_session() as session:
|
||||
author_col = session.query(Collection).filter_by(kind='author', name='Roland').one_or_none()
|
||||
assert author_col, 'Author collection should be re-created when doc is re-indexed'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_calibre_cover(test_session, async_client, test_directory, example_epub, example_mobi,
|
||||
image_file, refresh_files):
|
||||
"""Calibre cover near an ebook file is discovered."""
|
||||
metadata = test_directory / 'metadata.opf'
|
||||
metadata.touch()
|
||||
cover_image = test_directory / 'cover.jpg'
|
||||
shutil.move(image_file, cover_image)
|
||||
|
||||
await refresh_files()
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
assert doc.file_group.title == 'WROLPi Test Book'
|
||||
assert doc.cover_path
|
||||
assert {i['path'].name for i in doc.file_group.my_files()} == {'ebook example.epub', 'ebook example.mobi',
|
||||
'cover.jpg'}
|
||||
# Metadata and cover FileGroups were deleted.
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_normalizes_author(async_client, test_session, test_directory, example_epub, refresh_files):
|
||||
"""Author names are normalized before creating collections (e.g. quotes stripped)."""
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.db import get_db_session
|
||||
|
||||
# Patch extract_metadata to return a quoted author.
|
||||
from modules.docs.extractors import DocMetadata
|
||||
fake_metadata = DocMetadata(author='"Tony R. Kuphaldt"', title='Test Book')
|
||||
with patch('modules.docs.extract_metadata', return_value=fake_metadata):
|
||||
await refresh_files()
|
||||
|
||||
with get_db_session() as session:
|
||||
# Collection should have normalized name (no quotes).
|
||||
col = session.query(Collection).filter_by(kind='author', name='Tony R. Kuphaldt').one_or_none()
|
||||
assert col, 'Author collection should have normalized name without quotes'
|
||||
# No collection with the quoted name.
|
||||
quoted = session.query(Collection).filter_by(kind='author', name='"Tony R. Kuphaldt"').one_or_none()
|
||||
assert not quoted, 'Should not create collection with quoted author name'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_validates_subject(async_client, test_session, test_directory, example_epub, refresh_files):
|
||||
"""Junk subjects are rejected during modeling."""
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.db import get_db_session
|
||||
|
||||
from modules.docs.extractors import DocMetadata
|
||||
fake_metadata = DocMetadata(subject='01/2004', title='Test Book')
|
||||
with patch('modules.docs.extract_metadata', return_value=fake_metadata):
|
||||
await refresh_files()
|
||||
|
||||
with get_db_session() as session:
|
||||
col = session.query(Collection).filter_by(kind='subject', name='01/2004').one_or_none()
|
||||
assert not col, 'Should not create collection for date-like subject'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory,
|
||||
refresh_files):
|
||||
"""A cover file near the eBook is discovered."""
|
||||
cover_path = example_epub.with_suffix('.jpg')
|
||||
cover_path.write_bytes(image_bytes_factory())
|
||||
await refresh_files()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
assert doc.cover_path.read_bytes() == cover_path.read_bytes()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_extract_cover(test_session, test_directory, example_epub, refresh_files):
|
||||
"""First image is extracted from the Ebook and used as the cover."""
|
||||
await refresh_files()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
# The WROLPi logo is the first image in the example epub.
|
||||
assert doc.cover_path and doc.cover_path.stat().st_size == 297099
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory,
|
||||
refresh_files, await_background_tasks):
|
||||
"""An ebook is re-indexed when moved."""
|
||||
tag = await tag_factory()
|
||||
shutil.move(image_file, example_epub.with_suffix('.jpg'))
|
||||
await refresh_files()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
assert doc.cover_path
|
||||
assert_dict_contains(doc.file_group.data, dict(author='roland'))
|
||||
assert sorted([i['path'] for i in doc.file_group.my_files()]) == [
|
||||
test_directory / 'ebook example.epub',
|
||||
test_directory / 'ebook example.jpg',
|
||||
]
|
||||
doc.file_group.add_tag(test_session, tag.id)
|
||||
test_session.commit()
|
||||
|
||||
new_directory = test_directory / 'new'
|
||||
new_directory.mkdir()
|
||||
file_worker.queue_move(new_directory, [doc.file_group.primary_path])
|
||||
await await_background_tasks()
|
||||
test_session.expire_all()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
assert doc.file_group.primary_path == test_directory / 'new/ebook example.epub'
|
||||
assert doc.cover_path
|
||||
assert_dict_contains(doc.file_group.data, dict(author='roland'))
|
||||
assert sorted([i['path'] for i in doc.file_group.my_files()]) == [
|
||||
test_directory / 'new/ebook example.epub',
|
||||
test_directory / 'new/ebook example.jpg',
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_pdf_max_size(test_session, example_pdf):
|
||||
"""The contents of a large PDF are not indexed."""
|
||||
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
|
||||
|
||||
files_lib._upsert_files([example_pdf], now())
|
||||
|
||||
await doc_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
assert file_group.d_text is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_pdf_poster(test_session, example_pdf):
|
||||
"""PDF modeler finds an image with the same stem and uses it as the poster."""
|
||||
poster_path = example_pdf.with_suffix('.jpg')
|
||||
Image.new('RGB', (25, 25), color='grey').save(poster_path)
|
||||
|
||||
files_lib._upsert_files([example_pdf, poster_path], now())
|
||||
|
||||
await doc_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
assert file_group.indexed is True
|
||||
assert file_group.data is not None
|
||||
assert file_group.data.get('cover_path') == poster_path.name
|
||||
|
||||
|
||||
142
modules/docs/test/test_extractors.py
Normal file
142
modules/docs/test/test_extractors.py
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
import pytest
|
||||
|
||||
from modules.docs.extractors import extract_epub, extract_pdf, extract_metadata, extract_docx, extract_odt, \
|
||||
extract_doc, extract_cbz, DocMetadata
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_epub(test_session, test_directory, example_epub):
|
||||
"""Metadata can be extracted from an EPUB file."""
|
||||
metadata = extract_epub(example_epub)
|
||||
assert metadata.title == 'WROLPi Test Book'
|
||||
assert metadata.author == 'roland'
|
||||
assert metadata.text # Has text content.
|
||||
assert metadata.cover_bytes # Has a cover image.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_pdf(test_session, test_directory, example_pdf):
|
||||
"""Metadata can be extracted from a PDF file."""
|
||||
metadata = extract_pdf(example_pdf)
|
||||
# PDF may or may not have title metadata; at minimum it should return something.
|
||||
assert metadata is not None
|
||||
assert isinstance(metadata, DocMetadata)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_pdf_metadata(test_session, test_directory, example_pdf):
|
||||
"""PDF extractor extracts page count, date, cover, and text."""
|
||||
metadata = extract_pdf(example_pdf)
|
||||
assert metadata.title == 'WROLPi Test PDF'
|
||||
assert metadata.author == 'roland'
|
||||
assert metadata.page_count == 3
|
||||
assert metadata.published_date is not None
|
||||
assert metadata.cover_bytes
|
||||
assert metadata.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_docx(test_session, test_directory, example_docx):
|
||||
"""DOCX extractor extracts text content."""
|
||||
metadata = extract_docx(example_docx)
|
||||
assert isinstance(metadata, DocMetadata)
|
||||
assert metadata.title == 'Word Document'
|
||||
assert metadata.text
|
||||
assert 'Example Word Document' in metadata.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_odt(test_session, test_directory, example_odt):
|
||||
"""ODT extractor extracts title and text."""
|
||||
metadata = extract_odt(example_odt)
|
||||
assert metadata.title is not None
|
||||
assert metadata.text
|
||||
assert 'Test Header' in metadata.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_doc(test_session, test_directory, example_doc):
|
||||
"""DOC extractor extracts title from filename and text if catdoc/textutil available."""
|
||||
metadata = extract_doc(example_doc)
|
||||
assert metadata.title # At minimum, title from filename.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_cbz(test_session, test_directory, example_cbz):
|
||||
"""CBZ extractor extracts cover image."""
|
||||
metadata = extract_cbz(example_cbz)
|
||||
assert metadata.cover_bytes is not None
|
||||
assert len(metadata.cover_bytes) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_cbz_comicinfo(test_session, test_directory, example_cbz_metadata):
|
||||
"""CBZ extractor reads ComicInfo.xml metadata."""
|
||||
metadata = extract_cbz(example_cbz_metadata)
|
||||
assert metadata.title == 'Bobby Make-Believe #1'
|
||||
assert metadata.author == 'Frank King'
|
||||
assert metadata.publisher == 'Chicago Tribune'
|
||||
assert metadata.description == 'Bobby Make-Believe is a comic strip by Frank King.'
|
||||
assert metadata.language == 'en'
|
||||
assert metadata.subject == 'Comic Strip'
|
||||
assert metadata.page_count == 4
|
||||
assert metadata.published_date.year == 1915
|
||||
assert metadata.published_date.month == 6
|
||||
assert metadata.published_date.day == 15
|
||||
assert metadata.cover_bytes is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_cbt(test_session, test_directory, example_cbt):
|
||||
"""CBT (TAR) extractor extracts cover image."""
|
||||
metadata = extract_cbz(example_cbt)
|
||||
assert metadata.cover_bytes is not None
|
||||
assert len(metadata.cover_bytes) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_cbr(test_session, test_directory, example_cbr):
|
||||
"""CBR (RAR) extractor extracts cover image."""
|
||||
metadata = extract_cbz(example_cbr)
|
||||
assert metadata.cover_bytes is not None
|
||||
assert len(metadata.cover_bytes) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_cb7(test_session, test_directory, example_cb7):
|
||||
"""CB7 (7z) extractor extracts cover image."""
|
||||
metadata = extract_cbz(example_cb7)
|
||||
assert metadata.cover_bytes is not None
|
||||
assert len(metadata.cover_bytes) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_metadata_cbt(test_session, test_directory, example_cbt):
|
||||
"""extract_metadata dispatches CBT files to the comic extractor."""
|
||||
fg = FileGroup.from_paths(test_session, example_cbt)
|
||||
test_session.flush()
|
||||
metadata = extract_metadata(fg)
|
||||
assert metadata.cover_bytes is not None
|
||||
assert len(metadata.cover_bytes) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_metadata_cb7(test_session, test_directory, example_cb7):
|
||||
"""extract_metadata dispatches CB7 files to the comic extractor."""
|
||||
fg = FileGroup.from_paths(test_session, example_cb7)
|
||||
test_session.flush()
|
||||
metadata = extract_metadata(fg)
|
||||
assert metadata.cover_bytes is not None
|
||||
assert len(metadata.cover_bytes) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_metadata_dispatches(test_session, test_directory, example_epub):
|
||||
"""extract_metadata dispatches to the correct extractor based on mimetype."""
|
||||
fg = FileGroup.from_paths(test_session, example_epub)
|
||||
test_session.flush()
|
||||
|
||||
metadata = extract_metadata(fg)
|
||||
assert metadata.title == 'WROLPi Test Book'
|
||||
assert metadata.author == 'roland'
|
||||
250
modules/docs/test/test_lib.py
Normal file
250
modules/docs/test/test_lib.py
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
import pytest
|
||||
|
||||
from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \
|
||||
get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \
|
||||
search_authors_by_name, search_subjects_by_name
|
||||
|
||||
|
||||
def test_is_valid_author():
|
||||
"""Author validation rejects junk values."""
|
||||
assert is_valid_author('John Smith')
|
||||
assert is_valid_author('J.K. Rowling')
|
||||
assert is_valid_author('Dr. Seuss')
|
||||
|
||||
# Reject junk.
|
||||
assert not is_valid_author('')
|
||||
assert not is_valid_author(None)
|
||||
assert not is_valid_author('Administrator')
|
||||
assert not is_valid_author('administrator')
|
||||
assert not is_valid_author('Preferred Customer')
|
||||
assert not is_valid_author('Unknown')
|
||||
assert not is_valid_author('www.example.com')
|
||||
assert not is_valid_author('https://example.com')
|
||||
assert not is_valid_author('A') # Too short.
|
||||
assert not is_valid_author('x' * 101) # Too long.
|
||||
|
||||
# Reject year ranges and pure numbers.
|
||||
assert not is_valid_author('1483-1546')
|
||||
assert not is_valid_author('1564-1616')
|
||||
assert not is_valid_author('-1938')
|
||||
assert not is_valid_author('1851-1915')
|
||||
assert not is_valid_author('1114690')
|
||||
assert not is_valid_author('1588-1679')
|
||||
|
||||
# Reject strings with no letters.
|
||||
assert not is_valid_author('12345')
|
||||
assert not is_valid_author('---')
|
||||
|
||||
|
||||
def test_split_authors():
|
||||
"""Author strings with multiple authors are split correctly."""
|
||||
assert split_authors('John Smith') == ['John Smith']
|
||||
assert split_authors('John Smith; Jane Doe') == ['John Smith', 'Jane Doe']
|
||||
assert split_authors('John Smith & Jane Doe') == ['John Smith', 'Jane Doe']
|
||||
assert split_authors('') == []
|
||||
assert split_authors(None) == []
|
||||
|
||||
|
||||
def test_normalize_author():
|
||||
"""Author names are cleaned up before validation."""
|
||||
# Strip surrounding quotes.
|
||||
assert normalize_author('"Tony R. Kuphaldt"') == 'Tony R. Kuphaldt'
|
||||
assert normalize_author('"Mrs. D.A. Lincoln"') == 'Mrs. D.A. Lincoln'
|
||||
|
||||
# Remove bracket suffixes.
|
||||
assert normalize_author('"Mrs. A. J. Barnes." [from old catalog]') == 'Mrs. A. J. Barnes'
|
||||
assert normalize_author('Catherine] [from old catalog]') == 'Catherine'
|
||||
assert normalize_author('1851-1915. [from old catalog]') == '1851-1915'
|
||||
assert normalize_author('"Mrs. John Sterling," [from old catalog] ed') == 'Mrs. John Sterling'
|
||||
|
||||
# Remove trailing birth-death years.
|
||||
assert normalize_author('Caroline Althea (Stickney) 1843-1920') == 'Caroline Althea (Stickney)'
|
||||
assert normalize_author('1851-1917') == '1851-1917' # All numeric, nothing to strip — normalization passes through.
|
||||
|
||||
# Strip trailing period (but not initials).
|
||||
assert normalize_author('Carol Ann.') == 'Carol Ann'
|
||||
assert normalize_author('J.K.') == 'J.K.' # Initials stay.
|
||||
assert normalize_author('Dr. Seuss') == 'Dr. Seuss' # Mid-string period stays.
|
||||
|
||||
# Surrounding parentheses.
|
||||
assert normalize_author('(Thomas Joseph Workman)') == 'Thomas Joseph Workman'
|
||||
assert normalize_author('(Roland)') == 'Roland'
|
||||
|
||||
# Title-case normalization.
|
||||
assert normalize_author('john smith') == 'John Smith'
|
||||
assert normalize_author('JOHN SMITH') == 'John Smith'
|
||||
assert normalize_author('John Smith') == 'John Smith'
|
||||
assert normalize_author('j.k. rowling') == 'J.K. Rowling'
|
||||
|
||||
# Empty/None.
|
||||
assert normalize_author('') is None
|
||||
assert normalize_author(None) is None
|
||||
assert normalize_author(' ') is None
|
||||
|
||||
|
||||
def test_is_valid_subject():
|
||||
"""Subject validation rejects junk values."""
|
||||
# Valid subjects.
|
||||
assert is_valid_subject('Farming')
|
||||
assert is_valid_subject('Computer Science')
|
||||
assert is_valid_subject('First Aid')
|
||||
assert is_valid_subject('Science, Technology and Society')
|
||||
|
||||
# Reject empty/short/long.
|
||||
assert not is_valid_subject('')
|
||||
assert not is_valid_subject(None)
|
||||
assert not is_valid_subject('A')
|
||||
assert not is_valid_subject('x' * 151)
|
||||
|
||||
# Reject pure numeric/codes.
|
||||
assert not is_valid_subject('00110001')
|
||||
assert not is_valid_subject('062447')
|
||||
assert not is_valid_subject('105642-01')
|
||||
assert not is_valid_subject('106763-01')
|
||||
|
||||
# Reject date-like strings.
|
||||
assert not is_valid_subject('01/2004')
|
||||
assert not is_valid_subject('04/2005')
|
||||
assert not is_valid_subject('14 June 2006')
|
||||
|
||||
# Reject HTML entities.
|
||||
assert not is_valid_subject('something • else')
|
||||
|
||||
# Reject URLs.
|
||||
assert not is_valid_subject('https://example.com')
|
||||
assert not is_valid_subject('www.example.com')
|
||||
|
||||
# Reject junk names.
|
||||
assert not is_valid_subject('unknown')
|
||||
assert not is_valid_subject('n/a')
|
||||
|
||||
|
||||
def test_normalize_subject():
|
||||
"""Subject strings are cleaned up before validation."""
|
||||
# Strip whitespace.
|
||||
assert normalize_subject(' Farming ') == 'Farming'
|
||||
|
||||
# Decode HTML entities.
|
||||
assert normalize_subject('First & Second') == 'First & Second'
|
||||
|
||||
# Truncate long strings.
|
||||
long_subject = 'x' * 200
|
||||
assert len(normalize_subject(long_subject)) == 150
|
||||
|
||||
# Title-case normalization.
|
||||
assert normalize_subject('science fiction') == 'Science Fiction'
|
||||
assert normalize_subject('SCIENCE FICTION') == 'Science Fiction'
|
||||
assert normalize_subject('fairy tales') == 'Fairy Tales'
|
||||
assert normalize_subject('Fairy Tales') == 'Fairy Tales'
|
||||
|
||||
# Empty/None.
|
||||
assert normalize_subject('') is None
|
||||
assert normalize_subject(None) is None
|
||||
assert normalize_subject(' ') is None
|
||||
|
||||
|
||||
def test_split_subjects():
|
||||
"""Subject strings with multiple subjects are split correctly."""
|
||||
# Split on bare commas (no trailing space).
|
||||
assert split_subjects('First Aid,GENERAL PREP.,HEALTH') == ['First Aid', 'GENERAL PREP.', 'HEALTH']
|
||||
|
||||
# Split on semicolons.
|
||||
assert split_subjects('Science; Technology') == ['Science', 'Technology']
|
||||
|
||||
# Preserve commas followed by space (legitimate compound subjects).
|
||||
assert split_subjects('Science, Technology and Society') == ['Science, Technology and Society']
|
||||
|
||||
# Empty/None.
|
||||
assert split_subjects('') == []
|
||||
assert split_subjects(None) == []
|
||||
|
||||
|
||||
def test_get_or_create_author_collection(test_session):
|
||||
"""Author collections are created and reused."""
|
||||
col1 = get_or_create_author_collection(test_session, 'John Smith')
|
||||
assert col1 is not None
|
||||
assert col1.name == 'John Smith'
|
||||
assert col1.kind == 'author'
|
||||
|
||||
# Same author returns same collection.
|
||||
col2 = get_or_create_author_collection(test_session, 'John Smith')
|
||||
assert col1.id == col2.id
|
||||
|
||||
# Different author creates new collection.
|
||||
col3 = get_or_create_author_collection(test_session, 'Jane Doe')
|
||||
assert col3.id != col1.id
|
||||
|
||||
# Empty author returns None.
|
||||
assert get_or_create_author_collection(test_session, '') is None
|
||||
assert get_or_create_author_collection(test_session, None) is None
|
||||
|
||||
|
||||
def test_get_or_create_subject_collection(test_session):
|
||||
"""Subject collections are created and reused."""
|
||||
col1 = get_or_create_subject_collection(test_session, 'Farming')
|
||||
assert col1 is not None
|
||||
assert col1.name == 'Farming'
|
||||
assert col1.kind == 'subject'
|
||||
|
||||
col2 = get_or_create_subject_collection(test_session, 'Farming')
|
||||
assert col1.id == col2.id
|
||||
|
||||
assert get_or_create_subject_collection(test_session, '') is None
|
||||
|
||||
|
||||
def test_get_or_create_subject_collection_case_insensitive(test_session):
|
||||
"""Subject collections with different casing reuse the same collection."""
|
||||
col1 = get_or_create_subject_collection(test_session, 'Science Fiction')
|
||||
col2 = get_or_create_subject_collection(test_session, 'science fiction')
|
||||
col3 = get_or_create_subject_collection(test_session, 'SCIENCE FICTION')
|
||||
assert col1.id == col2.id == col3.id
|
||||
|
||||
|
||||
def test_get_or_create_author_collection_case_insensitive(test_session):
|
||||
"""Author collections with different casing reuse the same collection."""
|
||||
col1 = get_or_create_author_collection(test_session, 'John Smith')
|
||||
col2 = get_or_create_author_collection(test_session, 'john smith')
|
||||
col3 = get_or_create_author_collection(test_session, 'JOHN SMITH')
|
||||
assert col1.id == col2.id == col3.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_authors_by_name(test_session):
|
||||
"""Search for author collections by partial name."""
|
||||
get_or_create_author_collection(test_session, 'John Smith')
|
||||
get_or_create_author_collection(test_session, 'Jane Doe')
|
||||
get_or_create_author_collection(test_session, 'Johnny Appleseed')
|
||||
|
||||
# Partial match.
|
||||
results = await search_authors_by_name(test_session, 'john')
|
||||
assert len(results) == 2
|
||||
names = [r['name'] for r in results]
|
||||
assert 'John Smith' in names
|
||||
assert 'Johnny Appleseed' in names
|
||||
|
||||
# No match.
|
||||
results = await search_authors_by_name(test_session, 'nobody')
|
||||
assert len(results) == 0
|
||||
|
||||
# Limit.
|
||||
results = await search_authors_by_name(test_session, 'j', limit=1)
|
||||
assert len(results) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_subjects_by_name(test_session):
|
||||
"""Search for subject collections by partial name."""
|
||||
get_or_create_subject_collection(test_session, 'Science Fiction')
|
||||
get_or_create_subject_collection(test_session, 'Computer Science')
|
||||
get_or_create_subject_collection(test_session, 'History')
|
||||
|
||||
# Partial match.
|
||||
results = await search_subjects_by_name(test_session, 'science')
|
||||
assert len(results) == 2
|
||||
names = [r['name'] for r in results]
|
||||
assert 'Science Fiction' in names
|
||||
assert 'Computer Science' in names
|
||||
|
||||
# No match.
|
||||
results = await search_subjects_by_name(test_session, 'nothing')
|
||||
assert len(results) == 0
|
||||
56
modules/docs/test/test_models.py
Normal file
56
modules/docs/test/test_models.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
from modules.docs.models import Doc, DOC_MIMETYPES, mimetype_is_doc
|
||||
|
||||
|
||||
def test_doc_mimetypes():
|
||||
"""All expected mimetypes are included."""
|
||||
assert 'application/epub+zip' in DOC_MIMETYPES
|
||||
assert 'application/x-mobipocket-ebook' in DOC_MIMETYPES
|
||||
assert 'application/pdf' in DOC_MIMETYPES
|
||||
assert 'application/msword' in DOC_MIMETYPES
|
||||
assert 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' in DOC_MIMETYPES
|
||||
assert 'application/vnd.oasis.opendocument.text' in DOC_MIMETYPES
|
||||
assert 'application/x-cbz' in DOC_MIMETYPES
|
||||
assert 'application/x-cbr' in DOC_MIMETYPES
|
||||
|
||||
|
||||
def test_mimetype_is_doc():
|
||||
"""mimetype_is_doc correctly identifies doc mimetypes."""
|
||||
assert mimetype_is_doc('application/pdf')
|
||||
assert mimetype_is_doc('application/epub+zip')
|
||||
assert mimetype_is_doc('application/x-mobipocket-ebook')
|
||||
assert mimetype_is_doc('application/msword')
|
||||
assert not mimetype_is_doc('video/mp4')
|
||||
assert not mimetype_is_doc('text/plain')
|
||||
assert not mimetype_is_doc('image/jpeg')
|
||||
|
||||
|
||||
def test_doc_model(test_session, doc_factory):
|
||||
"""Doc model can be created and queried."""
|
||||
doc = doc_factory(size=1024, publisher='Test Publisher', language='en', subject='Testing')
|
||||
assert doc.id
|
||||
assert doc.size == 1024
|
||||
assert doc.publisher == 'Test Publisher'
|
||||
assert doc.language == 'en'
|
||||
assert doc.subject == 'Testing'
|
||||
assert doc.file_group is not None
|
||||
|
||||
queried = test_session.query(Doc).filter_by(id=doc.id).one()
|
||||
assert queried.publisher == 'Test Publisher'
|
||||
|
||||
|
||||
def test_doc_can_model(test_session, test_directory):
|
||||
"""Doc.can_model identifies correct FileGroups."""
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
pdf_path = test_directory / 'test.pdf'
|
||||
pdf_path.write_bytes(b'%PDF-1.4 test')
|
||||
fg = FileGroup.from_paths(test_session, pdf_path)
|
||||
test_session.flush()
|
||||
assert Doc.can_model(fg)
|
||||
|
||||
# Video should not be modeled as a doc.
|
||||
video_path = test_directory / 'test.mp4'
|
||||
video_path.write_bytes(b'\x00\x00\x00\x1cftypisom')
|
||||
fg2 = FileGroup.from_paths(test_session, video_path)
|
||||
test_session.flush()
|
||||
assert not Doc.can_model(fg2)
|
||||
|
|
@ -583,9 +583,10 @@ class Channel(ModelHelper, Base):
|
|||
collection = relationship('Collection', foreign_keys=[collection_id])
|
||||
|
||||
@property
|
||||
def downloads(self) -> InstrumentedList:
|
||||
"""Get downloads from associated Collection."""
|
||||
return self.collection.downloads if self.collection else []
|
||||
def downloads(self) -> list:
|
||||
"""Get recurring downloads from associated Collection. Once-downloads are excluded because they are not
|
||||
Channel downloads."""
|
||||
return [d for d in self.collection.downloads if d.frequency] if self.collection else []
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Channel id={self.id} name={repr(self.name)} url={self.url} directory={self.directory}>'
|
||||
|
|
|
|||
|
|
@ -398,6 +398,39 @@ async def test_import_channel_cleans_up_preexisting_orphaned_collections(await_s
|
|||
assert test_session.query(Collection).filter_by(id=orphan_id).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_once_download_excluded_from_channel_config(await_switches, test_session, channel_factory,
|
||||
test_channels_config):
|
||||
"""A once-download (no frequency) associated with a Channel's Collection should not be saved to the channels
|
||||
config. This prevents errors during import when a once-download has no frequency."""
|
||||
channel = channel_factory(download_frequency=DownloadFrequency.weekly)
|
||||
|
||||
# Create a once-download associated with the channel's collection (simulates a video download that got
|
||||
# associated with the channel via apply_yt_channel).
|
||||
once_download = Download(url='https://www.youtube.com/watch?v=once_video', downloader='video',
|
||||
collection_id=channel.collection_id)
|
||||
test_session.add(once_download)
|
||||
test_session.commit()
|
||||
# The once-download is in the Collection, but Channel.downloads filters it out.
|
||||
assert len(channel.collection.downloads) == 2
|
||||
assert len(channel.downloads) == 1
|
||||
|
||||
save_channels_config()
|
||||
await await_switches()
|
||||
|
||||
# The once-download should not appear in the config.
|
||||
config_text = test_channels_config.read_text()
|
||||
assert 'once_video' not in config_text
|
||||
# The recurring download should still be there.
|
||||
assert channel.url in config_text
|
||||
|
||||
# Importing should succeed without the "Refusing to create Download" error.
|
||||
test_session.delete(once_download)
|
||||
test_session.commit()
|
||||
import_channels_config()
|
||||
assert len(channel.downloads) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_channel_download_comments(await_switches, test_session, channel_factory,
|
||||
test_channels_config):
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ sysstat
|
|||
tar
|
||||
tcpdump
|
||||
tmux
|
||||
unrar-free
|
||||
unzip
|
||||
vim
|
||||
vlc
|
||||
|
|
|
|||
|
|
@ -52,7 +52,6 @@ chown -R _renderd:_renderd /var/cache/renderd/tiles
|
|||
/opt/wrolpi/scripts/install_help_service.sh
|
||||
|
||||
# Create the media directory for the wrolpi user.
|
||||
echo '/dev/sda1 /media/wrolpi auto defaults,nofail 0 0' | tee -a /etc/fstab
|
||||
mkdir -p /media/wrolpi
|
||||
chown -R wrolpi:wrolpi /media/wrolpi /home/wrolpi /opt/wrolpi*
|
||||
|
||||
|
|
|
|||
|
|
@ -19,10 +19,12 @@ PyMuPDF>=1.27.0
|
|||
pytest-asyncio==0.24.0
|
||||
pytest-xdist==3.5.0
|
||||
pytest==8.3.3
|
||||
py7zr>=1.1.0
|
||||
python-docx==1.1.2
|
||||
python-dotenv==1.0.1
|
||||
python-magic~=0.4.27
|
||||
pytz==2024.2
|
||||
rarfile>=4.2
|
||||
sanic-ext==25.12.0
|
||||
sanic-testing==24.6.0
|
||||
sanic==25.12.0
|
||||
|
|
|
|||
504
test/sample_reading_media/LICENSE
Normal file
504
test/sample_reading_media/LICENSE
Normal file
|
|
@ -0,0 +1,504 @@
|
|||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
Version 2.1, February 1999
|
||||
|
||||
Copyright (C) 1991, 1999 Free Software Foundation, Inc.
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
[This is the first released version of the Lesser GPL. It also counts
|
||||
as the successor of the GNU Library Public License, version 2, hence
|
||||
the version number 2.1.]
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
Licenses are intended to guarantee your freedom to share and change
|
||||
free software--to make sure the software is free for all its users.
|
||||
|
||||
This license, the Lesser General Public License, applies to some
|
||||
specially designated software packages--typically libraries--of the
|
||||
Free Software Foundation and other authors who decide to use it. You
|
||||
can use it too, but we suggest you first think carefully about whether
|
||||
this license or the ordinary General Public License is the better
|
||||
strategy to use in any particular case, based on the explanations below.
|
||||
|
||||
When we speak of free software, we are referring to freedom of use,
|
||||
not price. Our General Public Licenses are designed to make sure that
|
||||
you have the freedom to distribute copies of free software (and charge
|
||||
for this service if you wish); that you receive source code or can get
|
||||
it if you want it; that you can change the software and use pieces of
|
||||
it in new free programs; and that you are informed that you can do
|
||||
these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
distributors to deny you these rights or to ask you to surrender these
|
||||
rights. These restrictions translate to certain responsibilities for
|
||||
you if you distribute copies of the library or if you modify it.
|
||||
|
||||
For example, if you distribute copies of the library, whether gratis
|
||||
or for a fee, you must give the recipients all the rights that we gave
|
||||
you. You must make sure that they, too, receive or can get the source
|
||||
code. If you link other code with the library, you must provide
|
||||
complete object files to the recipients, so that they can relink them
|
||||
with the library after making changes to the library and recompiling
|
||||
it. And you must show them these terms so they know their rights.
|
||||
|
||||
We protect your rights with a two-step method: (1) we copyright the
|
||||
library, and (2) we offer you this license, which gives you legal
|
||||
permission to copy, distribute and/or modify the library.
|
||||
|
||||
To protect each distributor, we want to make it very clear that
|
||||
there is no warranty for the free library. Also, if the library is
|
||||
modified by someone else and passed on, the recipients should know
|
||||
that what they have is not the original version, so that the original
|
||||
author's reputation will not be affected by problems that might be
|
||||
introduced by others.
|
||||
|
||||
Finally, software patents pose a constant threat to the existence of
|
||||
any free program. We wish to make sure that a company cannot
|
||||
effectively restrict the users of a free program by obtaining a
|
||||
restrictive license from a patent holder. Therefore, we insist that
|
||||
any patent license obtained for a version of the library must be
|
||||
consistent with the full freedom of use specified in this license.
|
||||
|
||||
Most GNU software, including some libraries, is covered by the
|
||||
ordinary GNU General Public License. This license, the GNU Lesser
|
||||
General Public License, applies to certain designated libraries, and
|
||||
is quite different from the ordinary General Public License. We use
|
||||
this license for certain libraries in order to permit linking those
|
||||
libraries into non-free programs.
|
||||
|
||||
When a program is linked with a library, whether statically or using
|
||||
a shared library, the combination of the two is legally speaking a
|
||||
combined work, a derivative of the original library. The ordinary
|
||||
General Public License therefore permits such linking only if the
|
||||
entire combination fits its criteria of freedom. The Lesser General
|
||||
Public License permits more lax criteria for linking other code with
|
||||
the library.
|
||||
|
||||
We call this license the "Lesser" General Public License because it
|
||||
does Less to protect the user's freedom than the ordinary General
|
||||
Public License. It also provides other free software developers Less
|
||||
of an advantage over competing non-free programs. These disadvantages
|
||||
are the reason we use the ordinary General Public License for many
|
||||
libraries. However, the Lesser license provides advantages in certain
|
||||
special circumstances.
|
||||
|
||||
For example, on rare occasions, there may be a special need to
|
||||
encourage the widest possible use of a certain library, so that it becomes
|
||||
a de-facto standard. To achieve this, non-free programs must be
|
||||
allowed to use the library. A more frequent case is that a free
|
||||
library does the same job as widely used non-free libraries. In this
|
||||
case, there is little to gain by limiting the free library to free
|
||||
software only, so we use the Lesser General Public License.
|
||||
|
||||
In other cases, permission to use a particular library in non-free
|
||||
programs enables a greater number of people to use a large body of
|
||||
free software. For example, permission to use the GNU C Library in
|
||||
non-free programs enables many more people to use the whole GNU
|
||||
operating system, as well as its variant, the GNU/Linux operating
|
||||
system.
|
||||
|
||||
Although the Lesser General Public License is Less protective of the
|
||||
users' freedom, it does ensure that the user of a program that is
|
||||
linked with the Library has the freedom and the wherewithal to run
|
||||
that program using a modified version of the Library.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow. Pay close attention to the difference between a
|
||||
"work based on the library" and a "work that uses the library". The
|
||||
former contains code derived from the library, whereas the latter must
|
||||
be combined with the library in order to run.
|
||||
|
||||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License Agreement applies to any software library or other
|
||||
program which contains a notice placed by the copyright holder or
|
||||
other authorized party saying it may be distributed under the terms of
|
||||
this Lesser General Public License (also called "this License").
|
||||
Each licensee is addressed as "you".
|
||||
|
||||
A "library" means a collection of software functions and/or data
|
||||
prepared so as to be conveniently linked with application programs
|
||||
(which use some of those functions and data) to form executables.
|
||||
|
||||
The "Library", below, refers to any such software library or work
|
||||
which has been distributed under these terms. A "work based on the
|
||||
Library" means either the Library or any derivative work under
|
||||
copyright law: that is to say, a work containing the Library or a
|
||||
portion of it, either verbatim or with modifications and/or translated
|
||||
straightforwardly into another language. (Hereinafter, translation is
|
||||
included without limitation in the term "modification".)
|
||||
|
||||
"Source code" for a work means the preferred form of the work for
|
||||
making modifications to it. For a library, complete source code means
|
||||
all the source code for all modules it contains, plus any associated
|
||||
interface definition files, plus the scripts used to control compilation
|
||||
and installation of the library.
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running a program using the Library is not restricted, and output from
|
||||
such a program is covered only if its contents constitute a work based
|
||||
on the Library (independent of the use of the Library in a tool for
|
||||
writing it). Whether that is true depends on what the Library does
|
||||
and what the program that uses the Library does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Library's
|
||||
complete source code as you receive it, in any medium, provided that
|
||||
you conspicuously and appropriately publish on each copy an
|
||||
appropriate copyright notice and disclaimer of warranty; keep intact
|
||||
all the notices that refer to this License and to the absence of any
|
||||
warranty; and distribute a copy of this License along with the
|
||||
Library.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy,
|
||||
and you may at your option offer warranty protection in exchange for a
|
||||
fee.
|
||||
|
||||
2. You may modify your copy or copies of the Library or any portion
|
||||
of it, thus forming a work based on the Library, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) The modified work must itself be a software library.
|
||||
|
||||
b) You must cause the files modified to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
c) You must cause the whole of the work to be licensed at no
|
||||
charge to all third parties under the terms of this License.
|
||||
|
||||
d) If a facility in the modified Library refers to a function or a
|
||||
table of data to be supplied by an application program that uses
|
||||
the facility, other than as an argument passed when the facility
|
||||
is invoked, then you must make a good faith effort to ensure that,
|
||||
in the event an application does not supply such function or
|
||||
table, the facility still operates, and performs whatever part of
|
||||
its purpose remains meaningful.
|
||||
|
||||
(For example, a function in a library to compute square roots has
|
||||
a purpose that is entirely well-defined independent of the
|
||||
application. Therefore, Subsection 2d requires that any
|
||||
application-supplied function or table used by this function must
|
||||
be optional: if the application does not supply it, the square
|
||||
root function must still compute square roots.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Library,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Library, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote
|
||||
it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Library.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Library
|
||||
with the Library (or with a work based on the Library) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may opt to apply the terms of the ordinary GNU General Public
|
||||
License instead of this License to a given copy of the Library. To do
|
||||
this, you must alter all the notices that refer to this License, so
|
||||
that they refer to the ordinary GNU General Public License, version 2,
|
||||
instead of to this License. (If a newer version than version 2 of the
|
||||
ordinary GNU General Public License has appeared, then you can specify
|
||||
that version instead if you wish.) Do not make any other change in
|
||||
these notices.
|
||||
|
||||
Once this change is made in a given copy, it is irreversible for
|
||||
that copy, so the ordinary GNU General Public License applies to all
|
||||
subsequent copies and derivative works made from that copy.
|
||||
|
||||
This option is useful when you wish to copy part of the code of
|
||||
the Library into a program that is not a library.
|
||||
|
||||
4. You may copy and distribute the Library (or a portion or
|
||||
derivative of it, under Section 2) in object code or executable form
|
||||
under the terms of Sections 1 and 2 above provided that you accompany
|
||||
it with the complete corresponding machine-readable source code, which
|
||||
must be distributed under the terms of Sections 1 and 2 above on a
|
||||
medium customarily used for software interchange.
|
||||
|
||||
If distribution of object code is made by offering access to copy
|
||||
from a designated place, then offering equivalent access to copy the
|
||||
source code from the same place satisfies the requirement to
|
||||
distribute the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
5. A program that contains no derivative of any portion of the
|
||||
Library, but is designed to work with the Library by being compiled or
|
||||
linked with it, is called a "work that uses the Library". Such a
|
||||
work, in isolation, is not a derivative work of the Library, and
|
||||
therefore falls outside the scope of this License.
|
||||
|
||||
However, linking a "work that uses the Library" with the Library
|
||||
creates an executable that is a derivative of the Library (because it
|
||||
contains portions of the Library), rather than a "work that uses the
|
||||
library". The executable is therefore covered by this License.
|
||||
Section 6 states terms for distribution of such executables.
|
||||
|
||||
When a "work that uses the Library" uses material from a header file
|
||||
that is part of the Library, the object code for the work may be a
|
||||
derivative work of the Library even though the source code is not.
|
||||
Whether this is true is especially significant if the work can be
|
||||
linked without the Library, or if the work is itself a library. The
|
||||
threshold for this to be true is not precisely defined by law.
|
||||
|
||||
If such an object file uses only numerical parameters, data
|
||||
structure layouts and accessors, and small macros and small inline
|
||||
functions (ten lines or less in length), then the use of the object
|
||||
file is unrestricted, regardless of whether it is legally a derivative
|
||||
work. (Executables containing this object code plus portions of the
|
||||
Library will still fall under Section 6.)
|
||||
|
||||
Otherwise, if the work is a derivative of the Library, you may
|
||||
distribute the object code for the work under the terms of Section 6.
|
||||
Any executables containing that work also fall under Section 6,
|
||||
whether or not they are linked directly with the Library itself.
|
||||
|
||||
6. As an exception to the Sections above, you may also combine or
|
||||
link a "work that uses the Library" with the Library to produce a
|
||||
work containing portions of the Library, and distribute that work
|
||||
under terms of your choice, provided that the terms permit
|
||||
modification of the work for the customer's own use and reverse
|
||||
engineering for debugging such modifications.
|
||||
|
||||
You must give prominent notice with each copy of the work that the
|
||||
Library is used in it and that the Library and its use are covered by
|
||||
this License. You must supply a copy of this License. If the work
|
||||
during execution displays copyright notices, you must include the
|
||||
copyright notice for the Library among them, as well as a reference
|
||||
directing the user to the copy of this License. Also, you must do one
|
||||
of these things:
|
||||
|
||||
a) Accompany the work with the complete corresponding
|
||||
machine-readable source code for the Library including whatever
|
||||
changes were used in the work (which must be distributed under
|
||||
Sections 1 and 2 above); and, if the work is an executable linked
|
||||
with the Library, with the complete machine-readable "work that
|
||||
uses the Library", as object code and/or source code, so that the
|
||||
user can modify the Library and then relink to produce a modified
|
||||
executable containing the modified Library. (It is understood
|
||||
that the user who changes the contents of definitions files in the
|
||||
Library will not necessarily be able to recompile the application
|
||||
to use the modified definitions.)
|
||||
|
||||
b) Use a suitable shared library mechanism for linking with the
|
||||
Library. A suitable mechanism is one that (1) uses at run time a
|
||||
copy of the library already present on the user's computer system,
|
||||
rather than copying library functions into the executable, and (2)
|
||||
will operate properly with a modified version of the library, if
|
||||
the user installs one, as long as the modified version is
|
||||
interface-compatible with the version that the work was made with.
|
||||
|
||||
c) Accompany the work with a written offer, valid for at
|
||||
least three years, to give the same user the materials
|
||||
specified in Subsection 6a, above, for a charge no more
|
||||
than the cost of performing this distribution.
|
||||
|
||||
d) If distribution of the work is made by offering access to copy
|
||||
from a designated place, offer equivalent access to copy the above
|
||||
specified materials from the same place.
|
||||
|
||||
e) Verify that the user has already received a copy of these
|
||||
materials or that you have already sent this user a copy.
|
||||
|
||||
For an executable, the required form of the "work that uses the
|
||||
Library" must include any data and utility programs needed for
|
||||
reproducing the executable from it. However, as a special exception,
|
||||
the materials to be distributed need not include anything that is
|
||||
normally distributed (in either source or binary form) with the major
|
||||
components (compiler, kernel, and so on) of the operating system on
|
||||
which the executable runs, unless that component itself accompanies
|
||||
the executable.
|
||||
|
||||
It may happen that this requirement contradicts the license
|
||||
restrictions of other proprietary libraries that do not normally
|
||||
accompany the operating system. Such a contradiction means you cannot
|
||||
use both them and the Library together in an executable that you
|
||||
distribute.
|
||||
|
||||
7. You may place library facilities that are a work based on the
|
||||
Library side-by-side in a single library together with other library
|
||||
facilities not covered by this License, and distribute such a combined
|
||||
library, provided that the separate distribution of the work based on
|
||||
the Library and of the other library facilities is otherwise
|
||||
permitted, and provided that you do these two things:
|
||||
|
||||
a) Accompany the combined library with a copy of the same work
|
||||
based on the Library, uncombined with any other library
|
||||
facilities. This must be distributed under the terms of the
|
||||
Sections above.
|
||||
|
||||
b) Give prominent notice with the combined library of the fact
|
||||
that part of it is a work based on the Library, and explaining
|
||||
where to find the accompanying uncombined form of the same work.
|
||||
|
||||
8. You may not copy, modify, sublicense, link with, or distribute
|
||||
the Library except as expressly provided under this License. Any
|
||||
attempt otherwise to copy, modify, sublicense, link with, or
|
||||
distribute the Library is void, and will automatically terminate your
|
||||
rights under this License. However, parties who have received copies,
|
||||
or rights, from you under this License will not have their licenses
|
||||
terminated so long as such parties remain in full compliance.
|
||||
|
||||
9. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Library or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Library (or any work based on the
|
||||
Library), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Library or works based on it.
|
||||
|
||||
10. Each time you redistribute the Library (or any work based on the
|
||||
Library), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute, link with or modify the Library
|
||||
subject to these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties with
|
||||
this License.
|
||||
|
||||
11. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Library at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Library by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Library.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under any
|
||||
particular circumstance, the balance of the section is intended to apply,
|
||||
and the section as a whole is intended to apply in other circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
12. If the distribution and/or use of the Library is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Library under this License may add
|
||||
an explicit geographical distribution limitation excluding those countries,
|
||||
so that distribution is permitted only in or among countries not thus
|
||||
excluded. In such case, this License incorporates the limitation as if
|
||||
written in the body of this License.
|
||||
|
||||
13. The Free Software Foundation may publish revised and/or new
|
||||
versions of the Lesser General Public License from time to time.
|
||||
Such new versions will be similar in spirit to the present version,
|
||||
but may differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Library
|
||||
specifies a version number of this License which applies to it and
|
||||
"any later version", you have the option of following the terms and
|
||||
conditions either of that version or of any later version published by
|
||||
the Free Software Foundation. If the Library does not specify a
|
||||
license version number, you may choose any version ever published by
|
||||
the Free Software Foundation.
|
||||
|
||||
14. If you wish to incorporate parts of the Library into other free
|
||||
programs whose distribution conditions are incompatible with these,
|
||||
write to the author to ask for permission. For software which is
|
||||
copyrighted by the Free Software Foundation, write to the Free
|
||||
Software Foundation; we sometimes make exceptions for this. Our
|
||||
decision will be guided by the two goals of preserving the free status
|
||||
of all derivatives of our free software and of promoting the sharing
|
||||
and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
|
||||
WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
|
||||
EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
|
||||
OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
|
||||
KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
|
||||
LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
|
||||
THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
|
||||
WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
|
||||
AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
|
||||
FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
|
||||
CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
|
||||
LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
|
||||
RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
|
||||
FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
|
||||
SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
|
||||
DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Libraries
|
||||
|
||||
If you develop a new library, and you want it to be of the greatest
|
||||
possible use to the public, we recommend making it free software that
|
||||
everyone can redistribute and change. You can do so by permitting
|
||||
redistribution under these terms (or, alternatively, under the terms of the
|
||||
ordinary General Public License).
|
||||
|
||||
To apply these terms, attach the following notices to the library. It is
|
||||
safest to attach them to the start of each source file to most effectively
|
||||
convey the exclusion of warranty; and each file should have at least the
|
||||
"copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the library's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
|
||||
USA
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or your
|
||||
school, if any, to sign a "copyright disclaimer" for the library, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the
|
||||
library `Frob' (a library for tweaking knobs) written by James Random
|
||||
Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1990
|
||||
Ty Coon, President of Vice
|
||||
|
||||
That's all there is to it!
|
||||
81
test/sample_reading_media/README.md
Normal file
81
test/sample_reading_media/README.md
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
# sample_reading_media
|
||||
|
||||
Test sample ebooks, etc. Formats:
|
||||
|
||||
* test_book.md
|
||||
* test_book_md.md
|
||||
* test_book_txt.txt
|
||||
* test_book_txt_crlf_win.txt
|
||||
* test_book_txt_lf_unix.txt
|
||||
* test_book_html.html
|
||||
* test_book_odt.odt
|
||||
* test_book_docx.docx
|
||||
* test_book_pdf.pdf
|
||||
* test_book_fb2.fb2
|
||||
* test_book_epub.epub
|
||||
* test_book_md_zip.zip
|
||||
* test_book_txt_zip.zip
|
||||
* test_book_html_zip.zip
|
||||
* test_book_fb2_zip.zip
|
||||
* test_book_rtf.rtf
|
||||
* test_book_rtf_zip.zip
|
||||
|
||||
All the above sample books are generated from [test_book.md](./test_book.md).
|
||||
|
||||
* source_test_book_fb2.fb2
|
||||
* source_test_book_fb2_zip.fbz
|
||||
* source_test_book_fb2_dot_zip.fb2.zip
|
||||
* test_book_epub_more_detail.epub
|
||||
* test_book_pdf_more_detail.pdf
|
||||
|
||||
LGPL license, so feel free to use in/with your projects. If you modify them, share your changes.
|
||||
|
||||
Test sample comics, Formats:
|
||||
|
||||
* bobby_make_believe_sample.cb7
|
||||
* bobby_make_believe_sample.cbt
|
||||
* bobby_make_believe_sample.cbz
|
||||
* bobby_make_believe_sample.cbr
|
||||
* bobby_make_believe_sample_dir.cb7
|
||||
* bobby_make_believe_sample_dir.cbt
|
||||
* bobby_make_believe_sample_dir.cbz
|
||||
|
||||
|
||||
For Comic book also sample, see https://www.contrapositivediary.com/?p=1197
|
||||
(e.g. http://www.copperwood.com/pub/Elf%20Receiver%20Radio-Craft%20August%201936.cbz)
|
||||
|
||||
Images in images/bobby_make_believe/ are in the Public Domain and are the first 4 four pages of "Bobby Make-Believe (1915)" from https://comicbookplus.com/?dlid=26481
|
||||
|
||||
## Build setup
|
||||
|
||||
# Assuming Debian based
|
||||
sudo apt install pandoc zip
|
||||
sudo apt install wget
|
||||
sudo apt install p7zip-full
|
||||
sudo apt install rar
|
||||
|
||||
Issue build:
|
||||
|
||||
./build.sh
|
||||
|
||||
## TODO
|
||||
|
||||
* add images (e.g. to html, embedded and linked, epub, new md file with images etc.)
|
||||
* fb2 with title page/images
|
||||
* hand crafted html (e.g. include metadata)
|
||||
* mobi
|
||||
* prc/pdb?
|
||||
* azw
|
||||
* azw3
|
||||
* add comics
|
||||
* CBR
|
||||
* epub with images only
|
||||
* metadata for formats that support it
|
||||
* mobi
|
||||
* azw/azw3
|
||||
* Comic metadata; zip comment or a ComicInfo.xml - see
|
||||
* https://github.com/dickloraine/EmbedComicMetadata
|
||||
* https://github.com/comictagger/comictagger
|
||||
* https://code.google.com/archive/p/comicbookinfo/ (wiki)
|
||||
* https://wiki.mobileread.com/wiki/CBR_and_CBZ#Metadata
|
||||
|
||||
BIN
test/sample_reading_media/bobby_make_believe_cb7.cb7
Normal file
BIN
test/sample_reading_media/bobby_make_believe_cb7.cb7
Normal file
Binary file not shown.
BIN
test/sample_reading_media/bobby_make_believe_cbr.cbr
Normal file
BIN
test/sample_reading_media/bobby_make_believe_cbr.cbr
Normal file
Binary file not shown.
BIN
test/sample_reading_media/bobby_make_believe_cbt.cbt
Normal file
BIN
test/sample_reading_media/bobby_make_believe_cbt.cbt
Normal file
Binary file not shown.
BIN
test/sample_reading_media/bobby_make_believe_dir_cb7.cb7
Normal file
BIN
test/sample_reading_media/bobby_make_believe_dir_cb7.cb7
Normal file
Binary file not shown.
BIN
test/sample_reading_media/bobby_make_believe_dir_cbt.cbt
Normal file
BIN
test/sample_reading_media/bobby_make_believe_dir_cbt.cbt
Normal file
Binary file not shown.
BIN
test/sample_reading_media/bobby_make_believe_dir_cbz.cbz
Normal file
BIN
test/sample_reading_media/bobby_make_believe_dir_cbz.cbz
Normal file
Binary file not shown.
BIN
test/sample_reading_media/bobby_make_believe_metadata.cbz
Normal file
BIN
test/sample_reading_media/bobby_make_believe_metadata.cbz
Normal file
Binary file not shown.
BIN
test/sample_reading_media/bobby_make_believe_sample.cbz
Normal file
BIN
test/sample_reading_media/bobby_make_believe_sample.cbz
Normal file
Binary file not shown.
BIN
test/sample_reading_media/test_book_odt.odt
Normal file
BIN
test/sample_reading_media/test_book_odt.odt
Normal file
Binary file not shown.
|
|
@ -127,6 +127,7 @@ def get_installed_browsers() -> list:
|
|||
|
||||
return browsers
|
||||
|
||||
|
||||
# Videos
|
||||
YT_DLP_BIN = which(
|
||||
'/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Collection Library Functions
|
|||
Provides generic operations for collections of any kind (domains, channels, etc.).
|
||||
These functions are used by both the unified collection API and legacy endpoints.
|
||||
"""
|
||||
import asyncio
|
||||
import pathlib
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
from sqlalchemy import func
|
||||
|
|
@ -13,7 +13,6 @@ from sqlalchemy.orm import Session
|
|||
from wrolpi import flags
|
||||
from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
|
||||
from wrolpi.db import get_db_session
|
||||
import pathlib
|
||||
from wrolpi.errors import FileWorkerConflict
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
|
|
@ -68,6 +67,7 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
|
|||
# Separate collections by kind for batch processing
|
||||
domain_ids = [c.id for c in collections if c.kind == 'domain']
|
||||
channel_ids = [c.id for c in collections if c.kind == 'channel']
|
||||
generic_ids = [c.id for c in collections if c.kind not in ('domain', 'channel')]
|
||||
|
||||
# Batch query: Get archive stats for all domain collections at once
|
||||
domain_stats_map = {}
|
||||
|
|
@ -102,6 +102,19 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
|
|||
|
||||
channel_stats_map = {s.collection_id: s for s in channel_stats}
|
||||
|
||||
# Batch query: Get item counts for generic collections (author, subject, etc.)
|
||||
generic_stats_map = {}
|
||||
if generic_ids:
|
||||
from .models import CollectionItem
|
||||
generic_stats = session.query(
|
||||
CollectionItem.collection_id,
|
||||
func.count(CollectionItem.id).label('item_count'),
|
||||
).filter(
|
||||
CollectionItem.collection_id.in_(generic_ids)
|
||||
).group_by(CollectionItem.collection_id).all()
|
||||
|
||||
generic_stats_map = {s.collection_id: s.item_count for s in generic_stats}
|
||||
|
||||
# Convert to JSON and merge batch statistics
|
||||
result = []
|
||||
for collection in collections:
|
||||
|
|
@ -133,6 +146,10 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
|
|||
data['total_size'] = 0
|
||||
data['channel_id'] = None
|
||||
|
||||
else:
|
||||
# Generic collections (author, subject, etc.): use actual CollectionItem count.
|
||||
data['item_count'] = generic_stats_map.get(collection.id, 0)
|
||||
|
||||
result.append(data)
|
||||
|
||||
return result
|
||||
|
|
@ -198,6 +215,14 @@ def get_collection_with_stats(session: Session, collection_id: int) -> dict:
|
|||
data['video_count'] = 0
|
||||
data['total_size'] = 0
|
||||
|
||||
else:
|
||||
# Generic collections (author, subject, etc.): count actual CollectionItem rows.
|
||||
from .models import CollectionItem
|
||||
item_count = session.query(func.count(CollectionItem.id)).filter(
|
||||
CollectionItem.collection_id == collection_id
|
||||
).scalar()
|
||||
data['item_count'] = item_count or 0
|
||||
|
||||
return data
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import pathlib
|
||||
from typing import Optional, List
|
||||
|
||||
from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func, BigInteger, or_
|
||||
from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func, BigInteger, \
|
||||
or_
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
|
|
@ -11,9 +12,9 @@ from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_re
|
|||
from wrolpi.downloader import Download, save_downloads_config
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.lib import delete_directory
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.tags import Tag
|
||||
from wrolpi.tags import save_tags_config
|
||||
|
|
@ -58,7 +59,7 @@ def validate_collection_directory(directory: pathlib.Path) -> pathlib.Path:
|
|||
|
||||
class Collection(ModelHelper, Base):
|
||||
"""
|
||||
A Collection is a grouping of FileGroups (videos, archives, ebooks, etc).
|
||||
A Collection is a grouping of FileGroups (videos, archives, docs, etc).
|
||||
|
||||
Collections can be:
|
||||
- Directory-restricted: Only contains files within a specific directory tree
|
||||
|
|
@ -290,8 +291,8 @@ class Collection(ModelHelper, Base):
|
|||
tag_name = data.get('tag_name')
|
||||
file_format = data.get('file_format')
|
||||
kind = (data.get('kind') or 'channel').strip()
|
||||
# Validate known kinds (forward-compatible: allow future values)
|
||||
if kind not in {'channel', 'domain'}:
|
||||
# Validate known kinds using the registry (allows extensible kinds).
|
||||
if not collection_type_registry.is_registered(kind):
|
||||
logger.warning(f"Unknown collection kind '{kind}', defaulting to 'channel'")
|
||||
kind = 'channel'
|
||||
|
||||
|
|
@ -430,8 +431,8 @@ class Collection(ModelHelper, Base):
|
|||
file_format = data.get('file_format')
|
||||
kind = (data.get('kind') or 'channel').strip()
|
||||
|
||||
# Validate known kinds
|
||||
if kind not in {'channel', 'domain'}:
|
||||
# Validate known kinds using the registry.
|
||||
if not collection_type_registry.is_registered(kind):
|
||||
logger.warning(f"Unknown collection kind '{kind}', defaulting to 'channel'")
|
||||
kind = 'channel'
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from sqlalchemy.orm import Session, joinedload
|
|||
|
||||
from wrolpi.common import logger, get_media_directory
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
from .errors import UnknownCollection
|
||||
from .models import Collection
|
||||
|
||||
|
|
|
|||
|
|
@ -136,6 +136,34 @@ class TestCollectionsAPI:
|
|||
assert test_channel_data['total_size'] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_collections_includes_item_count_for_author(
|
||||
self, async_client, test_session, test_directory
|
||||
):
|
||||
"""Test that GET /api/collections?kind=author returns correct item_count from CollectionItem rows."""
|
||||
from modules.docs.lib import get_or_create_author_collection
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
# Create an author collection with a file group added to it.
|
||||
author_col = get_or_create_author_collection(test_session, 'John Smith')
|
||||
path = test_directory / 'test_doc.pdf'
|
||||
path.write_bytes(b'%PDF-1.4 test')
|
||||
fg = FileGroup.from_paths(test_session, path)
|
||||
test_session.flush()
|
||||
author_col.add_file_group(fg, session=test_session)
|
||||
test_session.commit()
|
||||
|
||||
# Make the request.
|
||||
request, response = await async_client.get('/api/collections?kind=author')
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
data = response.json
|
||||
|
||||
author_data = next(c for c in data['collections'] if c['name'] == 'John Smith')
|
||||
assert author_data['item_count'] == 1, \
|
||||
f'Author collection should report 1 item, got {author_data["item_count"]}'
|
||||
|
||||
|
||||
class TestCollectionTagInfoAPI:
|
||||
"""Test the POST /api/collections/<id>/tag_info endpoint."""
|
||||
|
||||
|
|
@ -415,7 +443,7 @@ class TestCollectionDeletion:
|
|||
self, async_client, test_session, test_directory, channel_factory, video_factory
|
||||
):
|
||||
"""Test that deleting a channel collection with videos orphans the videos."""
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.models import Channel
|
||||
|
||||
# Create a channel with videos
|
||||
channel = channel_factory(name='test', directory=test_directory / 'videos' / 'test')
|
||||
|
|
|
|||
|
|
@ -331,7 +331,6 @@ async def test_tag_domain_comprehensive(
|
|||
8. Tagging without directory doesn't move files
|
||||
"""
|
||||
from modules.archive.lib import save_domains_config, get_domains_config
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.common import walk
|
||||
|
||||
# Create domain collection directory in the usual archive directory
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ def test_collection_format_destination_domain(test_session, test_directory):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collection_format_destination_domain_with_tag_template(async_client, test_session, test_directory,
|
||||
test_wrolpi_config):
|
||||
test_wrolpi_config):
|
||||
"""Collection.format_destination uses tag when template includes %(domain_tag)s."""
|
||||
# Update config to include tag in template
|
||||
config = get_wrolpi_config()
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ from wrolpi.collections.reorganize import (
|
|||
get_batch_reorganization_status,
|
||||
ReorganizationPreview,
|
||||
)
|
||||
from wrolpi.common import get_media_directory
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
|
@ -874,7 +873,6 @@ async def test_reorganize_updates_tag_directory_links(async_client, test_directo
|
|||
async def test_batch_reorganize_preserves_tags(async_client, test_directory, video_factory, tag_factory,
|
||||
test_tags_config):
|
||||
"""Batch reorganization should preserve tags across multiple collections."""
|
||||
from wrolpi.files.worker import file_worker
|
||||
import asyncio
|
||||
|
||||
videos_dir = test_directory / 'videos'
|
||||
|
|
@ -1042,7 +1040,6 @@ async def test_reorganize_can_retry_after_partial_failure(async_client, test_dir
|
|||
- file_format should NOT have been updated (allowing retry)
|
||||
- needs_reorganization should still return True
|
||||
"""
|
||||
from wrolpi.files.worker import file_worker
|
||||
import shutil
|
||||
|
||||
# Create channel collection
|
||||
|
|
@ -1675,7 +1672,7 @@ def test_reorganize_skips_archives_missing_required_metadata(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_domain_reorganization_moves_files_from_root_to_year_subdirectory(
|
||||
async_client, test_directory, archive_factory
|
||||
async_client, test_directory, archive_factory
|
||||
):
|
||||
"""Archives in collection root should be moved to year subdirectories.
|
||||
|
||||
|
|
@ -1756,7 +1753,7 @@ async def test_domain_reorganization_moves_files_from_root_to_year_subdirectory(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_reorganization_moves_files_from_root_to_year_subdirectory(
|
||||
async_client, test_directory, video_factory
|
||||
async_client, test_directory, video_factory
|
||||
):
|
||||
"""Videos in collection root should be moved to year subdirectories.
|
||||
|
||||
|
|
@ -1843,7 +1840,7 @@ async def test_channel_reorganization_moves_files_from_root_to_year_subdirectory
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_reorganization_preserves_root_level_files(
|
||||
async_client, test_directory, video_factory
|
||||
async_client, test_directory, video_factory
|
||||
):
|
||||
"""Root-level files (like channel info.json and images) should NOT be moved during reorganization.
|
||||
|
||||
|
|
@ -2298,28 +2295,28 @@ async def test_conflict_preview_includes_quality_rank(async_client, test_directo
|
|||
# Filename with all fields: uploader_date_sourceid_title.ext
|
||||
# Should work with complex formats since filename parsing provides all metadata
|
||||
(
|
||||
"Learning Self-Reliance_20170529_p_MzsCFkUPU_How to Build a Solar Generator.mp4",
|
||||
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s",
|
||||
True, # Should work - filename has uploader, date, and source_id
|
||||
"Learning Self-Reliance_20170529_p_MzsCFkUPU_How to Build a Solar Generator.mp4",
|
||||
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s",
|
||||
True, # Should work - filename has uploader, date, and source_id
|
||||
),
|
||||
# Filename without source_id: uploader_date_title.ext pattern
|
||||
# Should NOT work with format requiring %(id)s
|
||||
(
|
||||
"Commsprepper_20130113_110Ah Car Battery connected to 80W Solar Panel.mp4",
|
||||
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s",
|
||||
False, # Should fail - filename doesn't have source_id and format requires it
|
||||
"Commsprepper_20130113_110Ah Car Battery connected to 80W Solar Panel.mp4",
|
||||
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s",
|
||||
False, # Should fail - filename doesn't have source_id and format requires it
|
||||
),
|
||||
# Filename without source_id but format doesn't require it
|
||||
(
|
||||
"Commsprepper_20130113_110Ah Car Battery.mp4",
|
||||
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s",
|
||||
False, # Should still fail - all-or-nothing: no source_id means no fallback
|
||||
"Commsprepper_20130113_110Ah Car Battery.mp4",
|
||||
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s",
|
||||
False, # Should still fail - all-or-nothing: no source_id means no fallback
|
||||
),
|
||||
# Simple title-only format (always works)
|
||||
(
|
||||
"random_video_name.mp4",
|
||||
"%(title)s.%(ext)s",
|
||||
True, # Should work - format only requires title
|
||||
"random_video_name.mp4",
|
||||
"%(title)s.%(ext)s",
|
||||
True, # Should work - format only requires title
|
||||
),
|
||||
])
|
||||
def test_video_metadata_fallback_from_filename(
|
||||
|
|
@ -2438,42 +2435,42 @@ def test_video_metadata_fallback_uses_channel_name_for_uploader(test_session, te
|
|||
# Complete filename with all fields - channel name takes priority over filename
|
||||
# This ensures reorganized files use the WROLPi channel name, not the YouTube uploader
|
||||
(
|
||||
"TestUploader_20170529_p_MzsCFkUPU_My Test Video.mp4",
|
||||
"FormatFallbackChannel",
|
||||
"My Test Video",
|
||||
None, # No DB datetime
|
||||
None, # No DB source_id
|
||||
"2017",
|
||||
"FormatFallbackChannel", # Channel name takes priority
|
||||
"p_MzsCFkUPU",
|
||||
["TestUploader"], # Should NOT contain filename uploader
|
||||
"TestUploader_20170529_p_MzsCFkUPU_My Test Video.mp4",
|
||||
"FormatFallbackChannel",
|
||||
"My Test Video",
|
||||
None, # No DB datetime
|
||||
None, # No DB source_id
|
||||
"2017",
|
||||
"FormatFallbackChannel", # Channel name takes priority
|
||||
"p_MzsCFkUPU",
|
||||
["TestUploader"], # Should NOT contain filename uploader
|
||||
),
|
||||
# Incomplete filename (no source_id) - uses DB values, not filename
|
||||
(
|
||||
"FilenameUploader_20200101_Just A Title.mp4",
|
||||
"ChannelFromDB",
|
||||
"Just A Title",
|
||||
"2024-06-15", # DB datetime
|
||||
"db_source_id", # DB source_id
|
||||
"2024",
|
||||
"ChannelFromDB",
|
||||
"db_source_id",
|
||||
["FilenameUploader", "2020"], # Should NOT contain these filename-parsed values
|
||||
"FilenameUploader_20200101_Just A Title.mp4",
|
||||
"ChannelFromDB",
|
||||
"Just A Title",
|
||||
"2024-06-15", # DB datetime
|
||||
"db_source_id", # DB source_id
|
||||
"2024",
|
||||
"ChannelFromDB",
|
||||
"db_source_id",
|
||||
["FilenameUploader", "2020"], # Should NOT contain these filename-parsed values
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_format_video_filename_fallback_behavior(
|
||||
test_session,
|
||||
test_directory,
|
||||
filename,
|
||||
channel_name,
|
||||
title,
|
||||
db_datetime,
|
||||
db_source_id,
|
||||
expected_year,
|
||||
expected_uploader,
|
||||
expected_source_id,
|
||||
should_not_contain,
|
||||
test_session,
|
||||
test_directory,
|
||||
filename,
|
||||
channel_name,
|
||||
title,
|
||||
db_datetime,
|
||||
db_source_id,
|
||||
expected_year,
|
||||
expected_uploader,
|
||||
expected_source_id,
|
||||
should_not_contain,
|
||||
):
|
||||
"""Test format_video_filename fallback behavior with various scenarios.
|
||||
|
||||
|
|
|
|||
|
|
@ -580,7 +580,6 @@ class ConfigFile:
|
|||
suffix = pathlib.Path(self.file_name).suffix
|
||||
return get_media_directory() / f'config/backup/{stem}-{backup_date}{suffix}'
|
||||
|
||||
|
||||
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
|
||||
"""Preview what a backup import would do. Subclasses override this."""
|
||||
raise NotImplementedError(f'{self.__class__.__name__} does not support backup import')
|
||||
|
|
|
|||
|
|
@ -249,7 +249,8 @@ class AutoAwaitTestClient:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[AutoAwaitTestClient, Any]:
|
||||
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[
|
||||
AutoAwaitTestClient, Any]:
|
||||
"""Get an Async Sanic Test Client with all default routes attached.
|
||||
|
||||
Depends on test_session to ensure the database mock is in place for session middleware.
|
||||
|
|
@ -536,6 +537,69 @@ def example_docx(test_directory) -> Generator[Path, Any, None]:
|
|||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_odt(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'test_book_odt.odt'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/test_book_odt.odt', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_cbz(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'bobby_make_believe_sample.cbz'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_sample.cbz', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_cbz_metadata(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'bobby_make_believe_metadata.cbz'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_metadata.cbz', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_cbt(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'bobby_make_believe_cbt.cbt'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbt.cbt', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_cbr(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'bobby_make_believe_cbr.cbr'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbr.cbr', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_cb7(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'bobby_make_believe_cb7.cb7'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cb7.cb7', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_cbz_dir(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'bobby_make_believe_dir_cbz.cbz'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbz.cbz', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_cbt_dir(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'bobby_make_believe_dir_cbt.cbt'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbt.cbt', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_cb7_dir(test_directory) -> Generator[Path, Any, None]:
|
||||
destination = test_directory / 'bobby_make_believe_dir_cb7.cb7'
|
||||
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cb7.cb7', destination)
|
||||
yield destination
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_files_structure(test_directory) -> Callable[[Union[List[Union[pathlib.Path, str]], dict]], List[pathlib.Path]]:
|
||||
"""
|
||||
|
|
@ -1038,5 +1102,3 @@ def refresh_files(async_client):
|
|||
await await_file_worker()
|
||||
|
||||
return _refresh
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ from http import HTTPStatus
|
|||
from itertools import filterfalse
|
||||
from typing import List, Dict, Generator, Iterable, Coroutine, Any
|
||||
from typing import Tuple, Optional
|
||||
from zoneinfo import ZoneInfo
|
||||
from urllib.parse import urlparse
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import feedparser
|
||||
import pytz
|
||||
|
|
@ -31,7 +31,6 @@ from sqlalchemy.sql import Delete
|
|||
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app, perpetual_signal
|
||||
|
||||
from wrolpi.cmd import which, run_command, CommandResult
|
||||
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
|
||||
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
|
||||
|
|
@ -1752,7 +1751,8 @@ class DownloadManagerConfig(ConfigFile):
|
|||
shutil.copy2(backup_file, config_file)
|
||||
elif mode == 'merge':
|
||||
backup_data = self.read_config_file(backup_file)
|
||||
current_data = self.read_config_file() if config_file.is_file() else dict(downloads=[], skip_urls=[], version=0)
|
||||
current_data = self.read_config_file() if config_file.is_file() else dict(downloads=[], skip_urls=[],
|
||||
version=0)
|
||||
|
||||
current_urls = {dl.get('url') for dl in current_data.get('downloads', []) if dl.get('url')}
|
||||
merged_downloads = list(current_data.get('downloads', []))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,2 @@
|
|||
from . import downloader # noqa
|
||||
from . import ebooks # noqa
|
||||
from . import pdfs # noqa
|
||||
from .api import files_bp
|
||||
|
|
|
|||
|
|
@ -11,11 +11,10 @@ from sanic_ext.extensions.openapi import openapi
|
|||
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory, logger, \
|
||||
background_task, walk, timer, TRACE_LEVEL, unique_by_predicate
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict, UnsupportedArchive
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
|
||||
from . import lib, schema
|
||||
from .worker import file_worker
|
||||
from ..api_utils import json_response, api_app
|
||||
from ..events import Events
|
||||
from ..schema import JSONErrorResponse
|
||||
from ..tags import Tag
|
||||
|
||||
|
|
|
|||
|
|
@ -1,350 +0,0 @@
|
|||
import asyncio
|
||||
import dataclasses
|
||||
import logging
|
||||
import pathlib
|
||||
from typing import Callable, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, or_, and_, Index
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
|
||||
from wrolpi.common import ModelHelper, Base, register_modeler, get_html_soup
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
try:
|
||||
import ebooklib
|
||||
from ebooklib import epub
|
||||
except ImportError:
|
||||
ebooklib = None
|
||||
epub = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
'EBOOK_MIMETYPES',
|
||||
'EBook',
|
||||
'EBookData',
|
||||
'EPUB_MIMETYPE',
|
||||
'MOBI_MIMETYPE',
|
||||
'ebook_modeler',
|
||||
'extract_ebook_cover',
|
||||
'extract_ebook_data',
|
||||
'mimetype_is_ebook',
|
||||
'model_ebook',
|
||||
]
|
||||
|
||||
EBOOK_SUFFIXES = ('.epub', '.mobi')
|
||||
EPUB_MIMETYPE = 'application/epub'
|
||||
MOBI_MIMETYPE = 'application/x-mobipocket-ebook'
|
||||
|
||||
EBOOK_MIMETYPES = (
|
||||
EPUB_MIMETYPE,
|
||||
MOBI_MIMETYPE,
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class EBookData:
|
||||
cover: bytes = None
|
||||
author: str = None
|
||||
text: str = None
|
||||
title: str = None
|
||||
ebook_path: pathlib.Path = None
|
||||
|
||||
def __bool__(self):
|
||||
return bool(self.cover) or bool(self.author) or bool(self.text) or bool(self.title)
|
||||
|
||||
def __json__(self) -> dict:
|
||||
d = {}
|
||||
if self.author:
|
||||
d['author'] = self.author
|
||||
if self.title:
|
||||
d['title'] = self.title
|
||||
if self.ebook_path:
|
||||
d['ebook_path'] = self.ebook_path
|
||||
return d
|
||||
|
||||
|
||||
def extract_text(html: str) -> str:
|
||||
soup = get_html_soup(html)
|
||||
text = soup.get_text()
|
||||
return text
|
||||
|
||||
|
||||
def mimetype_is_ebook(mimetype: str) -> bool:
|
||||
return any(i for i in EBOOK_MIMETYPES if mimetype.startswith(i))
|
||||
|
||||
|
||||
def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]:
|
||||
"""Extract data from within an eBook file."""
|
||||
if ebooklib is None or epub is None:
|
||||
raise ValueError('ebooklib is not installed')
|
||||
if not mimetype.startswith(EPUB_MIMETYPE):
|
||||
return None
|
||||
|
||||
data = EBookData(ebook_path=path)
|
||||
|
||||
book = epub.read_epub(path, options=dict(ignore_ncx=True))
|
||||
for key, value in book.metadata.items():
|
||||
if 'title' in value:
|
||||
data.title = data.title or value['title'][0][0]
|
||||
if 'creator' in value:
|
||||
data.author = data.author or value['creator'][0][0]
|
||||
|
||||
data.text = ''
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||||
data.text += extract_text(doc.content)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def extract_ebook_cover(path: pathlib.Path) -> Optional[bytes]:
|
||||
"""Attempt to find the Cover within an ePub file. If no cover is found, return the first image."""
|
||||
if ebooklib is None or epub is None:
|
||||
raise ValueError('ebooklib is not installed')
|
||||
|
||||
book = epub.read_epub(path, options=dict(ignore_ncx=True))
|
||||
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_COVER):
|
||||
logger.debug(f'Found cover for {path}')
|
||||
return doc.content
|
||||
|
||||
# Use the first image as the cover.
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_IMAGE):
|
||||
logger.debug(f'Using first image as cover for {path}')
|
||||
return doc.content
|
||||
|
||||
|
||||
def discover_calibre_cover(ebook_path: pathlib.Path):
|
||||
"""Calibre puts a cover file in a directory which contains only one ebook (but multiple formats of the ebook),
|
||||
this function will return the path of the cover if the ebook_path is in such a situation."""
|
||||
if not ebook_path.is_file():
|
||||
raise ValueError(f'Invalid ebook path {ebook_path}')
|
||||
|
||||
paths = list(ebook_path.parent.iterdir())
|
||||
|
||||
ebook_path_stem, _ = split_path_stem_and_suffix(ebook_path)
|
||||
|
||||
cover = None
|
||||
for path in paths:
|
||||
stem, suffix = split_path_stem_and_suffix(path)
|
||||
if path.is_dir():
|
||||
# We don't care about sub directories.
|
||||
continue
|
||||
if stem == ebook_path_stem:
|
||||
# This is one of the formats of the ebook.
|
||||
continue
|
||||
if path.name == 'metadata.opf':
|
||||
# Metadata, this is expected in a Calibre book directory.
|
||||
continue
|
||||
if stem == 'cover' and get_mimetype(path).startswith('image/'):
|
||||
# Discovered the cover.
|
||||
cover = path
|
||||
continue
|
||||
# Some file that does not match the ebook stem, metadata, or cover. This must not be a Calibre book directory.
|
||||
return None
|
||||
return cover
|
||||
|
||||
|
||||
class EBook(ModelHelper, Base):
|
||||
__tablename__ = 'ebook'
|
||||
__table_args__ = (
|
||||
Index('ebook_size_idx', 'size'),
|
||||
)
|
||||
id = Column(Integer, primary_key=True)
|
||||
size: int = Column(Integer)
|
||||
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
def __repr__(self):
|
||||
path = str(self.file_group.primary_path)
|
||||
return f'<EBook id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
|
||||
|
||||
def __json__(self) -> dict:
|
||||
# Convert this eBook's paths to pathlib.Paths, resolving relative to directory.
|
||||
d = self.file_group.__json__()
|
||||
if d.get('data') and d['data'].get('ebook_path'):
|
||||
d['data']['ebook_path'] = self.file_group.resolve_path(d['data']['ebook_path'])
|
||||
if d.get('data') and d['data'].get('cover_path'):
|
||||
d['data']['cover_path'] = self.file_group.resolve_path(d['data']['cover_path'])
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def can_model(file_group: FileGroup) -> bool:
|
||||
if mimetype_is_ebook(file_group.mimetype):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def do_model(session: Session, file_group: FileGroup) -> 'EBook':
|
||||
ebook = model_ebook(file_group, session)
|
||||
file_group.indexed = True
|
||||
return ebook
|
||||
|
||||
def generate_cover(self) -> Optional[pathlib.Path]:
|
||||
"""Discover this ebook's cover, if found, store it next to the ebook file."""
|
||||
path = self.file_group.primary_path
|
||||
|
||||
if not path:
|
||||
raise ValueError('Cannot generate cover for ebook without ebook file!')
|
||||
|
||||
if calibre_cover := discover_calibre_cover(path):
|
||||
self.hide_calibre_files()
|
||||
return calibre_cover
|
||||
|
||||
cover_bytes = extract_ebook_cover(path)
|
||||
if cover_bytes:
|
||||
cover_path = path.with_suffix('.jpeg')
|
||||
cover_path.write_bytes(cover_bytes)
|
||||
return cover_path
|
||||
|
||||
logger.warning(f'Unable to generate ebook cover for {self.file_group.primary_path}')
|
||||
|
||||
@property
|
||||
def cover_file(self) -> Optional[dict]:
|
||||
posters = self.file_group.my_poster_files()
|
||||
return posters[0]
|
||||
|
||||
@property
|
||||
def cover_path(self) -> Optional[pathlib.Path]:
|
||||
return self.cover_file['path']
|
||||
|
||||
def hide_calibre_files(self):
|
||||
"""Delete FileGroups of the cover/metadata when this is a Calibre ebook directory."""
|
||||
path = self.file_group.primary_path
|
||||
if not discover_calibre_cover(path):
|
||||
raise ValueError(f'Refusing to hide Calibre files when we do not have a Calibre ebook directory!')
|
||||
|
||||
metadata = path.parent / 'metadata.opf'
|
||||
cover = path.parent / 'cover.jpg'
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup) \
|
||||
.filter(or_(
|
||||
FileGroup.primary_path == str(metadata),
|
||||
FileGroup.primary_path == str(cover),
|
||||
))
|
||||
for file_group in file_groups:
|
||||
session.delete(file_group)
|
||||
|
||||
|
||||
def model_ebook(file_group: FileGroup, session: Session) -> EBook:
|
||||
"""Creates an EBook model based off a File. Searches for its cover in the provided `files`."""
|
||||
# Check if an EBook already exists for this FileGroup
|
||||
ebook = session.query(EBook).filter_by(file_group_id=file_group.id).one_or_none()
|
||||
if not ebook:
|
||||
# Create new EBook if it doesn't exist
|
||||
ebook = EBook(file_group=file_group)
|
||||
session.add(ebook)
|
||||
|
||||
# Multiple formats may be available.
|
||||
epub_file = next(iter(ebook.file_group.my_epub_files()), None)
|
||||
mobi_file = next(iter(ebook.file_group.my_mobi_files()), None)
|
||||
# epub is preferred because it can be indexed.
|
||||
if epub_file:
|
||||
ebook_file = epub_file['path']
|
||||
ebook_mimetype = epub_file['mimetype']
|
||||
elif mobi_file:
|
||||
ebook_file = mobi_file['path']
|
||||
ebook_mimetype = mobi_file['mimetype']
|
||||
else:
|
||||
raise ValueError(f'No epub or mobi found. {ebook}')
|
||||
|
||||
size = ebook_file.stat().st_size
|
||||
|
||||
# Only index if it hasn't been done, or if the file has changed.
|
||||
changed = size != ebook.size or not ebook.file_group.title
|
||||
if epub_file and changed:
|
||||
# Only read the contents of the file if it has changed.
|
||||
try:
|
||||
data = extract_ebook_data(ebook_file, ebook_mimetype)
|
||||
if data:
|
||||
# Title is a_text.
|
||||
ebook.file_group.title = ebook.file_group.a_text = data.title
|
||||
# Author is b_text.
|
||||
ebook.file_group.b_text = ebook.file_group.author = data.author
|
||||
# All text is d_text.
|
||||
ebook.file_group.d_text = data.text
|
||||
d: dict = data.__json__()
|
||||
# Store ebook_path as filename only (relative to directory)
|
||||
d['ebook_path'] = ebook_file.name
|
||||
ebook.file_group.data = d
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract epub book data', exc_info=e)
|
||||
|
||||
if ebook.file_group.my_poster_files():
|
||||
# Store cover_path as filename only (relative to directory)
|
||||
cover_path = ebook.file_group.my_poster_files()[0]['path']
|
||||
ebook.file_group.data['cover_path'] = cover_path.name
|
||||
else:
|
||||
# Extract cover because there is no cover file.
|
||||
cover_path = ebook.generate_cover()
|
||||
if cover_path:
|
||||
ebook.file_group.append_files(cover_path)
|
||||
# Store cover_path as filename only (relative to directory)
|
||||
ebook.file_group.data['cover_path'] = cover_path.name
|
||||
|
||||
if not ebook.file_group.title:
|
||||
# Book was not indexed above (probably a MOBI), use the file data.
|
||||
stem, _ = split_path_stem_and_suffix(ebook.file_group.primary_path)
|
||||
ebook.file_group.title = stem
|
||||
|
||||
ebook.size = size
|
||||
ebook.flush()
|
||||
|
||||
return ebook
|
||||
|
||||
|
||||
def find_ebook_files_in_group(group: List):
|
||||
"""Returns a list of any File records if they are an ebook."""
|
||||
return list(filter(lambda i: mimetype_is_ebook(i.mimetype), group))
|
||||
|
||||
|
||||
@register_modeler
|
||||
async def ebook_modeler(progress_callback: Callable[[int], None] = None):
|
||||
"""Searches for ebook files and models them into the ebook table."""
|
||||
total_processed = 0
|
||||
while True:
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup, EBook).filter(
|
||||
or_(
|
||||
FileGroup.mimetype == 'application/epub+zip',
|
||||
FileGroup.mimetype == 'application/x-mobipocket-ebook',
|
||||
),
|
||||
FileGroup.indexed == False,
|
||||
).outerjoin(EBook, and_(EBook.file_group_id == FileGroup.id)) \
|
||||
.limit(10)
|
||||
file_groups: List[Tuple[FileGroup, EBook]] = list(file_groups)
|
||||
|
||||
processed = 0
|
||||
for file_group, ebook in file_groups:
|
||||
processed += 1
|
||||
try:
|
||||
if PYTEST:
|
||||
# Handle caching issue during testing.
|
||||
session.expire(file_group)
|
||||
ebook = ebook or model_ebook(file_group, session)
|
||||
session.add(ebook)
|
||||
file_group.model = EBook.__tablename__
|
||||
file_group.indexed = True
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index ebook {file_group}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
session.commit()
|
||||
|
||||
# Report batch progress
|
||||
total_processed += len(file_groups)
|
||||
if progress_callback and len(file_groups) > 0:
|
||||
progress_callback(total_processed)
|
||||
|
||||
if processed < 10:
|
||||
# Did not reach limit, no more books.
|
||||
break
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
|
@ -8,8 +8,8 @@ from zipfile import ZipFile
|
|||
import docx
|
||||
|
||||
from wrolpi.cmd import CATDOC_PATH, TEXTUTIL_PATH
|
||||
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
|
||||
from wrolpi.common import logger, split_lines_by_length, truncate_object_bytes, extract_html_text, get_title_from_html
|
||||
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -274,7 +274,7 @@ async def delete(*paths: Union[str, pathlib.Path]):
|
|||
|
||||
def _mimetype_suffix_map(path: Path, mimetype: str):
|
||||
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
|
||||
from wrolpi.files.ebooks import MOBI_MIMETYPE
|
||||
from modules.docs.models import MOBI_MIMETYPE
|
||||
suffix = path.suffix.lower()
|
||||
if mimetype == 'application/octet-stream':
|
||||
if suffix.endswith('.mobi'):
|
||||
|
|
@ -315,6 +315,12 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
|
|||
return 'text/srt'
|
||||
if mimetype == 'application/zip' and suffix == '.docx':
|
||||
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||||
if mimetype == 'application/zip' and suffix == '.cbz':
|
||||
return 'application/x-cbz'
|
||||
if mimetype == 'application/zip' and suffix == '.odt':
|
||||
return 'application/vnd.oasis.opendocument.text'
|
||||
if mimetype in ('application/x-rar', 'application/x-rar-compressed') and suffix == '.cbr':
|
||||
return 'application/x-cbr'
|
||||
if mimetype == 'video/x-ms-asf' and suffix == '.wma':
|
||||
return 'audio/x-ms-wma'
|
||||
return mimetype
|
||||
|
|
@ -530,6 +536,14 @@ def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]])
|
|||
if mimetype.startswith('application/x-mobipocket-ebook') and not has_epub:
|
||||
# Only return a MOBI if no EPUB is present.
|
||||
return file
|
||||
if mimetype in ('application/msword',
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'application/vnd.oasis.opendocument.text',
|
||||
'application/x-cbz', 'application/x-cbr',
|
||||
'application/vnd.comicbook+zip', 'application/vnd.comicbook-rar'):
|
||||
return file
|
||||
if file.suffix.lower() in ('.cbz', '.cbr', '.cbt', '.cb7'):
|
||||
return file
|
||||
|
||||
# Secondary file types.
|
||||
for file, mimetype in file_mimetypes:
|
||||
|
|
@ -1440,19 +1454,19 @@ def _bulk_update_file_groups_reorganize(updates: List[dict]):
|
|||
for update in updates:
|
||||
# Don't change indexed - reorganization only moves files, content unchanged
|
||||
curs.execute('''
|
||||
UPDATE file_group
|
||||
SET directory = %s,
|
||||
primary_path = %s,
|
||||
files = %s,
|
||||
data = %s
|
||||
WHERE id = %s
|
||||
''', (
|
||||
update['directory'],
|
||||
update['primary_path'],
|
||||
json.dumps(update['files'], default=_json_serial),
|
||||
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
|
||||
update['id'],
|
||||
))
|
||||
UPDATE file_group
|
||||
SET directory = %s,
|
||||
primary_path = %s,
|
||||
files = %s,
|
||||
data = %s
|
||||
WHERE id = %s
|
||||
''', (
|
||||
update['directory'],
|
||||
update['primary_path'],
|
||||
json.dumps(update['files'], default=_json_serial),
|
||||
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
|
||||
update['id'],
|
||||
))
|
||||
|
||||
# Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates.
|
||||
# This is necessary because get_db_curs uses the same connection as get_db_session
|
||||
|
|
@ -2085,12 +2099,16 @@ bulk_tag_worker = perpetual_signal(sleep=1)(bulk_tag_worker)
|
|||
|
||||
|
||||
def _detect_archive_type(path: pathlib.Path) -> str:
|
||||
"""Detect archive type from file suffix. Returns 'zip', 'tar', or raises UnsupportedArchive."""
|
||||
"""Detect archive type from file suffix. Returns 'zip', 'tar', 'rar', '7z', or raises UnsupportedArchive."""
|
||||
name = path.name.lower()
|
||||
if name.endswith('.zip'):
|
||||
if name.endswith(('.zip', '.cbz')):
|
||||
return 'zip'
|
||||
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz')):
|
||||
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz', '.cbt')):
|
||||
return 'tar'
|
||||
if name.endswith(('.rar', '.cbr')):
|
||||
return 'rar'
|
||||
if name.endswith(('.7z', '.cb7')):
|
||||
return '7z'
|
||||
raise UnsupportedArchive(f'Unsupported archive format: {path.name}')
|
||||
|
||||
|
||||
|
|
@ -2198,6 +2216,36 @@ def list_archive_contents(path: pathlib.Path) -> dict:
|
|||
if not is_dir:
|
||||
total_size += member.size
|
||||
total_files += 1
|
||||
elif archive_type == 'rar':
|
||||
import rarfile
|
||||
with rarfile.RarFile(path, 'r') as rf:
|
||||
for info in rf.infolist():
|
||||
is_dir = info.is_dir()
|
||||
entry = {
|
||||
'path': info.filename,
|
||||
'name': info.filename.rstrip('/').split('/')[-1],
|
||||
'is_dir': is_dir,
|
||||
'size': info.file_size,
|
||||
}
|
||||
flat_entries.append(entry)
|
||||
if not is_dir:
|
||||
total_size += info.file_size
|
||||
total_files += 1
|
||||
elif archive_type == '7z':
|
||||
import py7zr
|
||||
with py7zr.SevenZipFile(path, 'r') as sz:
|
||||
for entry_info in sz.list():
|
||||
is_dir = entry_info.is_directory
|
||||
entry = {
|
||||
'path': entry_info.filename + ('/' if is_dir else ''),
|
||||
'name': entry_info.filename.rstrip('/').split('/')[-1],
|
||||
'is_dir': is_dir,
|
||||
'size': entry_info.uncompressed if not is_dir else 0,
|
||||
}
|
||||
flat_entries.append(entry)
|
||||
if not is_dir:
|
||||
total_size += entry_info.uncompressed
|
||||
total_files += 1
|
||||
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
|
||||
raise UnsupportedArchive(f'Cannot read archive: {e}')
|
||||
|
||||
|
|
@ -2247,6 +2295,28 @@ def stream_archive_member(archive_path: pathlib.Path, member_path: str) -> Tuple
|
|||
if f is None:
|
||||
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
|
||||
file_bytes = f.read()
|
||||
elif archive_type == 'rar':
|
||||
import rarfile
|
||||
with rarfile.RarFile(archive_path, 'r') as rf:
|
||||
try:
|
||||
info = rf.getinfo(member_path)
|
||||
except KeyError:
|
||||
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
||||
if info.is_dir():
|
||||
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
|
||||
file_bytes = rf.read(member_path)
|
||||
elif archive_type == '7z':
|
||||
import py7zr
|
||||
import tempfile
|
||||
with py7zr.SevenZipFile(archive_path, 'r') as sz:
|
||||
if member_path not in sz.getnames():
|
||||
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
sz.extract(tmpdir, targets=[member_path])
|
||||
extracted_path = pathlib.Path(tmpdir) / member_path
|
||||
if not extracted_path.is_file():
|
||||
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
|
||||
file_bytes = extracted_path.read_bytes()
|
||||
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
|
||||
raise UnsupportedArchive(f'Cannot read archive: {e}')
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
|
|
@ -98,7 +98,7 @@ class FileGroup(ModelHelper, Base):
|
|||
indexed = Column(Boolean, default=lambda: False) # wrolpi.files.lib.apply_indexers
|
||||
length = Column(BigInteger) # video duration, article words, etc.
|
||||
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
|
||||
model = Column(String) # "video", "archive", "ebook", etc.
|
||||
model = Column(String) # "video", "archive", "doc", etc.
|
||||
modification_datetime = Column(TZDateTime) # the modification date of the file on disk
|
||||
primary_path: pathlib.Path = Column(MediaPathType, nullable=False, unique=True)
|
||||
published_datetime = Column(TZDateTime) # the date the creator published this file
|
||||
|
|
@ -551,7 +551,6 @@ class FileGroup(ModelHelper, Base):
|
|||
This method first checks if the path is a primary_path, and if not,
|
||||
searches for a FileGroup where the file is in the files list.
|
||||
"""
|
||||
from sqlalchemy import text
|
||||
|
||||
path = pathlib.Path(path)
|
||||
directory = str(path.parent)
|
||||
|
|
@ -725,13 +724,13 @@ class FileGroup(ModelHelper, Base):
|
|||
"""Return the class that is suitable for this FileGroup (usually based on mimetype), if any."""
|
||||
from modules.videos.models import Video
|
||||
from modules.archive.models import Archive
|
||||
from wrolpi.files.ebooks import EBook
|
||||
from modules.docs.models import Doc
|
||||
from modules.zim.models import Zim
|
||||
|
||||
if Video.can_model(self):
|
||||
return Video
|
||||
elif EBook.can_model(self):
|
||||
return EBook
|
||||
elif Doc.can_model(self):
|
||||
return Doc
|
||||
elif Archive.can_model(self):
|
||||
return Archive
|
||||
elif Zim.can_model(self):
|
||||
|
|
|
|||
|
|
@ -1,191 +0,0 @@
|
|||
import asyncio
|
||||
import dataclasses
|
||||
import pathlib
|
||||
from typing import Callable, Generator
|
||||
|
||||
import pytz
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.common import register_modeler, logger, truncate_generator_bytes, truncate_object_bytes, \
|
||||
split_lines_by_length, slow_logger
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files.indexers import Indexer
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import FILE_MAX_PDF_SIZE, PYTEST, FILE_MAX_TEXT_SIZE
|
||||
|
||||
try:
|
||||
import pymupdf
|
||||
except ImportError:
|
||||
pymupdf = None
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['get_pdf_metadata', 'get_words', 'pdf_modeler']
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PDFMetadata:
|
||||
title: str = None
|
||||
author: str = None
|
||||
published_datetime: str = None
|
||||
modification_datetime: str = None
|
||||
|
||||
|
||||
def _parse_pdf_date(date_str: str):
|
||||
"""Parse PDF date format 'D:YYYYMMDDHHmmSS+HH'mm'' to datetime."""
|
||||
if not date_str:
|
||||
return None
|
||||
try:
|
||||
parsed = dates.strpdate(date_str)
|
||||
if parsed and not parsed.tzinfo:
|
||||
parsed = parsed.astimezone(pytz.UTC)
|
||||
return parsed
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_pdf_metadata(doc, path: pathlib.Path) -> PDFMetadata:
|
||||
"""Extract title/author from the PDF metadata."""
|
||||
data = PDFMetadata()
|
||||
|
||||
if doc is None or doc.metadata is None:
|
||||
logger.error(f'Cannot get title of {path} PyMuPDF is not installed.')
|
||||
data.title = Indexer.get_title(path)
|
||||
return data
|
||||
|
||||
metadata = doc.metadata
|
||||
|
||||
if (title := metadata.get('title')) and title.lower() != 'unknown':
|
||||
data.title = title
|
||||
if (author := metadata.get('author')) and author.lower() != 'unknown':
|
||||
data.author = author
|
||||
|
||||
if creation_date := metadata.get('creationDate'):
|
||||
data.published_datetime = _parse_pdf_date(creation_date)
|
||||
|
||||
if mod_date := metadata.get('modDate'):
|
||||
data.modification_datetime = _parse_pdf_date(mod_date)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def get_words(doc, path: pathlib.Path) -> Generator[str, None, None]:
|
||||
"""
|
||||
Reads all text in a PDF.
|
||||
|
||||
Note, may return more than can be stored in Postgres. See: truncate_generator_bytes
|
||||
"""
|
||||
if pymupdf is None:
|
||||
logger.error(f'Cannot index {path} PyMuPDF is not installed.')
|
||||
yield ''
|
||||
return
|
||||
|
||||
for page in doc:
|
||||
text = page.get_text()
|
||||
# Postgres does not allow null characters.
|
||||
text = text.replace('\x00', '\uFFFD').strip()
|
||||
if text:
|
||||
yield text
|
||||
|
||||
|
||||
# PDFs can be large, limit how many are modeled at once.
|
||||
PDF_PROCESSING_LIMIT = 10
|
||||
|
||||
|
||||
@register_modeler
|
||||
async def pdf_modeler(progress_callback: Callable[[int], None] = None):
|
||||
"""Queries for any PDF files that have not been indexed. Each PDF found will be indexed."""
|
||||
if not pymupdf:
|
||||
logger.warning(f'Cannot index PDF without PyMuPDF')
|
||||
return
|
||||
|
||||
total_processed = 0
|
||||
while True:
|
||||
# Continually query for PDFs that have not been indexed.
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup) \
|
||||
.filter(
|
||||
# Get all groups that contain a PDF that have not been indexed.
|
||||
FileGroup.indexed != True,
|
||||
FileGroup.mimetype == 'application/pdf',
|
||||
).limit(PDF_PROCESSING_LIMIT)
|
||||
file_groups = list(file_groups)
|
||||
|
||||
processed = 0
|
||||
for file_group in file_groups:
|
||||
processed += 1
|
||||
pdf_files = file_group.my_files('application/pdf')
|
||||
if not pdf_files:
|
||||
logger.error('Query returned a group without a PDF!')
|
||||
continue
|
||||
|
||||
# Assume the first PDF is the only PDF.
|
||||
pdf_file = pdf_files[0]['path']
|
||||
doc = None
|
||||
try:
|
||||
file_title = Indexer.get_title(pdf_file)
|
||||
|
||||
doc = pymupdf.open(pdf_file)
|
||||
|
||||
with slow_logger(2, f'Modeling PDF took %(elapsed)s seconds: {file_group}',
|
||||
logger__=logger):
|
||||
metadata = get_pdf_metadata(doc, pdf_file)
|
||||
|
||||
words = ''
|
||||
if pdf_file.stat().st_size > FILE_MAX_PDF_SIZE:
|
||||
logger.warning(f'PDF too large to fully index: {pdf_file}')
|
||||
else:
|
||||
try:
|
||||
# PDFs are complex, don't fail to create title index if text extraction fails.
|
||||
words = '\n'.join(
|
||||
truncate_generator_bytes(get_words(doc, pdf_file), FILE_MAX_TEXT_SIZE))
|
||||
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
|
||||
words = split_lines_by_length(words)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index {pdf_file}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
file_group.title = file_group.a_text = metadata.title or file_title
|
||||
file_group.author = file_group.b_text = metadata.author
|
||||
file_group.c_text = file_title # The name of the file may not match the title in the PDF metadata.
|
||||
file_group.d_text = words or None
|
||||
file_group.published_datetime = metadata.published_datetime
|
||||
file_group.published_modified_datetime = metadata.modification_datetime
|
||||
file_group.model = 'pdf'
|
||||
|
||||
# Find an image with the same stem to use as the poster.
|
||||
# Store as filename only (consistent with other models, enables fast moves/renames).
|
||||
poster_files = file_group.my_poster_files()
|
||||
if poster_files:
|
||||
file_group.data = file_group.data or {}
|
||||
file_group.data['poster_path'] = poster_files[0]['path'].name
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index PDF {pdf_file}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
finally:
|
||||
if doc:
|
||||
doc.close()
|
||||
|
||||
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
|
||||
file_group.indexed = True
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
logger.info(f'pdf_modeler processed {processed} PDFs')
|
||||
|
||||
session.commit()
|
||||
|
||||
# Report batch progress
|
||||
total_processed += len(file_groups)
|
||||
if progress_callback and len(file_groups) > 0:
|
||||
progress_callback(total_processed)
|
||||
|
||||
if processed < PDF_PROCESSING_LIMIT:
|
||||
# Did not reach limit, do not query again.
|
||||
break
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
|
@ -103,5 +103,3 @@ class BulkTagApplyRequest:
|
|||
@dataclass
|
||||
class ArchiveContentsRequest:
|
||||
path: str
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import pytest
|
|||
|
||||
from modules.videos import Video
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.files import lib
|
||||
from wrolpi.files.lib import get_mimetype
|
||||
from wrolpi.files.models import FileGroup, Directory
|
||||
from wrolpi.tags import TagFile
|
||||
|
|
@ -337,7 +336,7 @@ async def test_file_statistics(test_session, async_client, test_directory, examp
|
|||
'tagged_files': 0,
|
||||
'tagged_zims': 0,
|
||||
'tags_count': 0,
|
||||
'total_count': 6, # extracted cover + .ffprobe.json
|
||||
'total_count': 7, # extracted covers (epub + pdf) + .ffprobe.json
|
||||
'video_count': 1,
|
||||
'zip_count': 0,
|
||||
}
|
||||
|
|
@ -696,7 +695,7 @@ async def test_post_upload(test_session, async_client, test_directory, make_file
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_upload_video_extracts_ffprobe(test_session, async_client, test_directory, make_files_structure,
|
||||
make_multipart_form, video_bytes):
|
||||
make_multipart_form, video_bytes):
|
||||
"""Uploading a video file should extract and store ffprobe data."""
|
||||
make_files_structure(['uploads/'])
|
||||
|
||||
|
|
@ -929,7 +928,7 @@ async def test_rename_file(test_session, test_directory, make_files_structure, a
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_file_with_associated_files_via_api(test_session, test_directory, make_files_structure,
|
||||
async_client, video_bytes, srt_text, refresh_files):
|
||||
async_client, video_bytes, srt_text, refresh_files):
|
||||
"""Renaming a FileGroup via API should rename all associated files.
|
||||
|
||||
Regression test: When renaming "example.mp4" to "example 2.mp4" and back via API,
|
||||
|
|
@ -1272,7 +1271,7 @@ async def test_get_file_skip_tracking(test_session, async_client, test_directory
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_file_ignored_directory_not_tracked(test_session, async_client, test_directory, make_files_structure,
|
||||
refresh_files, test_wrolpi_config, await_switches):
|
||||
refresh_files, test_wrolpi_config, await_switches):
|
||||
"""Files in ignored directories do not get their viewed timestamp set."""
|
||||
# Create files in a directory that will be ignored
|
||||
foo, bar = make_files_structure({'config/settings.txt': 'settings', 'normal/file.txt': 'contents'})
|
||||
|
|
|
|||
|
|
@ -240,3 +240,103 @@ async def test_download_traversal_attack(test_session, async_client, test_direct
|
|||
_, response = await async_client.get('/api/files/zip/download?path=test.zip&member=../../../etc/passwd')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert 'INVALID_ARCHIVE_MEMBER' in response.text
|
||||
|
||||
|
||||
# ---- Comic book archive format tests using real sample files ----
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_cbt_contents(test_session, async_client, test_directory, example_cbt):
|
||||
"""CBT (TAR) archive contents can be listed."""
|
||||
_, response = await async_client.post('/api/files/zip/contents',
|
||||
content=json.dumps({'path': example_cbt.name}))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
contents = response.json['contents']
|
||||
assert contents['total_files'] == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_cbt_dir_contents(test_session, async_client, test_directory, example_cbt_dir):
|
||||
"""CBT (TAR) archive with nested directories can be listed."""
|
||||
_, response = await async_client.post('/api/files/zip/contents',
|
||||
content=json.dumps({'path': example_cbt_dir.name}))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
contents = response.json['contents']
|
||||
assert contents['total_files'] == 4
|
||||
# Files are in a nested directory.
|
||||
entry_names = [e['name'] for e in contents['entries']]
|
||||
assert 'images' in entry_names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_cbt_member(test_session, async_client, test_directory, example_cbt):
|
||||
"""A single image can be downloaded from a CBT archive."""
|
||||
_, response = await async_client.get(
|
||||
f'/api/files/zip/download?path={example_cbt.name}&member=Bobby-Make-Believe_1915__0.jpg')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.body) > 0
|
||||
assert response.body[:2] == b'\xff\xd8' # JPEG magic bytes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_cbr_contents(test_session, async_client, test_directory, example_cbr):
|
||||
"""CBR (RAR) archive contents can be listed."""
|
||||
_, response = await async_client.post('/api/files/zip/contents',
|
||||
content=json.dumps({'path': example_cbr.name}))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
contents = response.json['contents']
|
||||
assert contents['total_files'] == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_cbr_member(test_session, async_client, test_directory, example_cbr):
|
||||
"""A single image can be downloaded from a CBR archive."""
|
||||
_, response = await async_client.get(
|
||||
f'/api/files/zip/download?path={example_cbr.name}&member=Bobby-Make-Believe_1915__0.jpg')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.body) > 0
|
||||
assert response.body[:2] == b'\xff\xd8' # JPEG magic bytes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_cb7_contents(test_session, async_client, test_directory, example_cb7):
|
||||
"""CB7 (7z) archive contents can be listed."""
|
||||
_, response = await async_client.post('/api/files/zip/contents',
|
||||
content=json.dumps({'path': example_cb7.name}))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
contents = response.json['contents']
|
||||
assert contents['total_files'] == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_cb7_dir_contents(test_session, async_client, test_directory, example_cb7_dir):
|
||||
"""CB7 (7z) archive with nested directories can be listed."""
|
||||
_, response = await async_client.post('/api/files/zip/contents',
|
||||
content=json.dumps({'path': example_cb7_dir.name}))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
contents = response.json['contents']
|
||||
assert contents['total_files'] == 4
|
||||
entry_names = [e['name'] for e in contents['entries']]
|
||||
assert 'images' in entry_names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_cb7_member(test_session, async_client, test_directory, example_cb7):
|
||||
"""A single image can be downloaded from a CB7 archive."""
|
||||
_, response = await async_client.get(
|
||||
f'/api/files/zip/download?path={example_cb7.name}&member=Bobby-Make-Believe_1915__0.jpg')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.body) > 0
|
||||
assert response.body[:2] == b'\xff\xd8' # JPEG magic bytes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_cbz_dir_contents(test_session, async_client, test_directory, example_cbz_dir):
|
||||
"""CBZ (ZIP) archive with nested directories can be listed."""
|
||||
_, response = await async_client.post('/api/files/zip/contents',
|
||||
content=json.dumps({'path': example_cbz_dir.name}))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
contents = response.json['contents']
|
||||
assert contents['total_files'] == 4
|
||||
entry_names = [e['name'] for e in contents['entries']]
|
||||
assert 'images' in entry_names
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
"""Test for ebook_modeler loop to ensure it processes more than one batch.
|
||||
|
||||
This test ensures the ebook_modeler can process more than its batch limit (10)
|
||||
files. If the loop logic has an off-by-one error (like using enumerate() with wrong
|
||||
comparison), the modeler would only process one batch and exit early.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.files.ebooks import ebook_modeler, EBook
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
# The ebook_modeler uses a hardcoded limit of 10
|
||||
EBOOK_PROCESSING_LIMIT = 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ebook_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory):
|
||||
"""
|
||||
Test that ebook_modeler processes MORE than the batch limit (10) files.
|
||||
|
||||
This test creates 15 ebook files and verifies that ebook_modeler
|
||||
processes all of them, not just the first batch of 10.
|
||||
|
||||
This catches off-by-one bugs in the loop logic (e.g., using enumerate()
|
||||
which is 0-indexed but comparing against the limit incorrectly).
|
||||
"""
|
||||
ebook_dir = test_directory / 'ebooks'
|
||||
ebook_dir.mkdir(parents=True)
|
||||
|
||||
# Create more ebooks than the batch limit
|
||||
num_ebooks = EBOOK_PROCESSING_LIMIT + 5 # 15 ebooks
|
||||
ebook_paths = []
|
||||
|
||||
for i in range(num_ebooks):
|
||||
ebook_path = ebook_dir / f'test_ebook_{i:03d}.epub'
|
||||
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', ebook_path)
|
||||
ebook_paths.append(ebook_path)
|
||||
|
||||
# Create FileGroups for each ebook file (simulating what refresh does)
|
||||
for ebook_path in ebook_paths:
|
||||
fg = FileGroup.from_paths(test_session, ebook_path)
|
||||
# EPUB mimetype might be 'application/epub+zip' or 'application/epub'
|
||||
assert fg.mimetype.startswith('application/epub')
|
||||
|
||||
test_session.commit()
|
||||
|
||||
# Verify we have the expected number of unindexed ebook FileGroups
|
||||
unindexed_count = test_session.query(FileGroup).filter(
|
||||
FileGroup.indexed == False,
|
||||
FileGroup.mimetype == 'application/epub+zip',
|
||||
).count()
|
||||
assert unindexed_count == num_ebooks, f"Expected {num_ebooks} unindexed ebook files, got {unindexed_count}"
|
||||
|
||||
# Run the ebook_modeler
|
||||
await ebook_modeler()
|
||||
|
||||
# Count how many EBooks were created
|
||||
ebook_count = test_session.query(EBook).count()
|
||||
|
||||
# All ebooks should be created
|
||||
assert ebook_count == num_ebooks, \
|
||||
f"ebook_modeler should process ALL {num_ebooks} files, but only processed {ebook_count}. " \
|
||||
f"This may be an off-by-one bug in the loop logic!"
|
||||
|
||||
# Also verify all FileGroups are now indexed
|
||||
still_unindexed = test_session.query(FileGroup).filter(
|
||||
FileGroup.indexed == False,
|
||||
FileGroup.mimetype == 'application/epub+zip',
|
||||
).count()
|
||||
assert still_unindexed == 0, \
|
||||
f"All ebook FileGroups should be indexed, but {still_unindexed} remain unindexed"
|
||||
|
|
@ -1,175 +0,0 @@
|
|||
import json
|
||||
import shutil
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.files.ebooks import EBook, EPUB_MIMETYPE
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index(async_client, test_session, test_directory, example_epub, example_mobi, refresh_files):
|
||||
"""Ebooks can be indexed.
|
||||
|
||||
Covers can be discovered."""
|
||||
await refresh_files()
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
assert ebook.file_group.mimetype.startswith(EPUB_MIMETYPE)
|
||||
assert ebook.file_group.author == 'roland'
|
||||
assert ebook.size == 292579
|
||||
# data now stores relative filenames only (not absolute paths)
|
||||
assert ebook.file_group.data == {'author': 'roland', 'title': 'WROLPi Test Book',
|
||||
'cover_path': 'ebook example.jpeg',
|
||||
'ebook_path': 'ebook example.epub',
|
||||
}
|
||||
|
||||
assert ebook.file_group.a_text, 'Book title was not indexed'
|
||||
assert ebook.file_group.b_text, 'Book author was not indexed'
|
||||
assert ebook.file_group.d_text, 'Book text was not indexed'
|
||||
# Both ebooks are assumed to be the same book, but different formats.
|
||||
assert (epubs := ebook.file_group.my_files('application/epub+zip')) and len(epubs) == 1 \
|
||||
and epubs[0]['path'] == example_epub
|
||||
assert (mobis := ebook.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1 \
|
||||
and mobis[0]['path'] == example_mobi
|
||||
# Cover was discovered.
|
||||
assert len(ebook.file_group.my_poster_files()) == 1
|
||||
# EPUB, MOBI, JPEG.
|
||||
assert len(ebook.file_group.files) == 3
|
||||
|
||||
# Ebooks can be deleted during refresh.
|
||||
example_epub.unlink()
|
||||
await refresh_files()
|
||||
assert test_session.query(EBook).count() == 1
|
||||
ebook = test_session.query(EBook).one()
|
||||
assert not ebook.file_group.my_files('application/epub+zip')
|
||||
assert (mobis := ebook.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1 \
|
||||
and mobis[0]['path'] == example_mobi
|
||||
# Cover was discovered.
|
||||
assert len(ebook.file_group.my_poster_files()) == 1
|
||||
# MOBI, JPEG.
|
||||
assert len(ebook.file_group.files) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory, await_switches,
|
||||
refresh_files):
|
||||
cover_path = example_epub.with_suffix('.jpg')
|
||||
cover_path.write_bytes(image_bytes_factory())
|
||||
await refresh_files()
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
|
||||
# Cover file near the eBook was discovered.
|
||||
assert ebook.cover_path.read_bytes() == cover_path.read_bytes()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_cover(test_session, test_directory, example_epub, await_switches, refresh_files):
|
||||
"""First image is extracted from the Ebook and used as the cover."""
|
||||
await refresh_files()
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
# The WROLPi logo is the first image in the example epub.
|
||||
assert ebook.cover_path and ebook.cover_path.stat().st_size == 297099
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_ebooks(test_session, async_client, example_epub):
|
||||
"""Ebooks are handled in File search results."""
|
||||
request, response = await async_client.post('/api/files/refresh')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
assert test_session.query(EBook).count() == 1
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
assert ebook.file_group.author == 'roland'
|
||||
assert ebook.file_group.a_text == 'WROLPi Test Book', 'Book title was not updated'
|
||||
assert ebook.file_group.d_text, 'Book was not indexed'
|
||||
|
||||
content = dict(mimetypes=['application/epub', 'application/x-mobipocket-ebook'])
|
||||
request, response = await async_client.post('/api/files/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json
|
||||
file_group = response.json['file_groups'][0]
|
||||
epub_file = file_group['files'][0]
|
||||
assert epub_file['path'] == 'ebook example.epub' and epub_file['mimetype'] == 'application/epub+zip'
|
||||
assert_dict_contains(
|
||||
file_group['data'],
|
||||
{'cover_path': 'ebook example.jpeg', 'ebook_path': 'ebook example.epub', 'title': 'WROLPi Test Book',
|
||||
'author': 'roland'},
|
||||
)
|
||||
|
||||
# No Mobi ebook.
|
||||
content = dict(mimetypes=['application/x-mobipocket-ebook'])
|
||||
request, response = await async_client.post('/api/files/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json
|
||||
assert len(response.json['file_groups']) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_calibre_cover(test_session, async_client, test_directory, example_epub, example_mobi,
|
||||
image_file, refresh_files):
|
||||
"""Calibre puts a cover near an ebook file, test if it can be found."""
|
||||
# Create a Calibre metadata file.
|
||||
metadata = test_directory / 'metadata.opf'
|
||||
metadata.touch()
|
||||
|
||||
# Create a "cover.jpg" file near the ebook, this is the Calibre cover.
|
||||
cover_image = test_directory / 'cover.jpg'
|
||||
shutil.move(image_file, cover_image)
|
||||
|
||||
# Calibre cover image was discovered, no cover was generated.
|
||||
await refresh_files()
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
assert ebook.cover_file['path'] == test_directory / 'cover.jpg'
|
||||
# Use my_files() to get resolved Path objects
|
||||
assert {i['path'].name for i in ebook.file_group.my_files()} == {'ebook example.epub', 'ebook example.mobi', 'cover.jpg'}
|
||||
# Cover is not the first image from the EPUB.
|
||||
assert ebook.cover_path.stat().st_size != 292579
|
||||
# Metadata and cover were deleted from the FileGroups.
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory,
|
||||
refresh_files, await_background_tasks):
|
||||
"""An ebook is re-indexed when moved."""
|
||||
tag = await tag_factory()
|
||||
shutil.move(image_file, example_epub.with_suffix('.jpg'))
|
||||
await refresh_files()
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.cover_path
|
||||
assert_dict_contains(ebook.file_group.data, dict(author='roland'))
|
||||
# Use my_files() to get resolved absolute Path objects
|
||||
assert sorted([i['path'] for i in ebook.file_group.my_files()]) == [
|
||||
test_directory / 'ebook example.epub',
|
||||
test_directory / 'ebook example.jpg',
|
||||
]
|
||||
ebook.file_group.add_tag(test_session, tag.id)
|
||||
test_session.commit()
|
||||
|
||||
new_directory = test_directory / 'new'
|
||||
new_directory.mkdir()
|
||||
file_worker.queue_move(new_directory, [ebook.file_group.primary_path])
|
||||
await await_background_tasks()
|
||||
test_session.expire_all()
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.file_group.primary_path == test_directory / 'new/ebook example.epub'
|
||||
assert ebook.cover_path
|
||||
assert_dict_contains(ebook.file_group.data, dict(author='roland'))
|
||||
# Use my_files() to get resolved absolute Path objects
|
||||
assert sorted([i['path'] for i in ebook.file_group.my_files()]) == [
|
||||
test_directory / 'new/ebook example.epub',
|
||||
test_directory / 'new/ebook example.jpg',
|
||||
]
|
||||
|
|
@ -158,7 +158,7 @@ async def test_delete_similar_names(async_client, test_session, make_files_struc
|
|||
])
|
||||
|
||||
await lib.delete('config/._wrolpi.yaml', 'config/._wrolpi.yaml.sb-98fa34a1-MG7phT',
|
||||
'config/._wrolpi.yaml.sb-98fa34a1-N4k8NB')
|
||||
'config/._wrolpi.yaml.sb-98fa34a1-N4k8NB')
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -948,7 +948,7 @@ async def test_doc_indexer(async_client, test_session, example_doc, refresh_file
|
|||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
doc = test_session.query(FileGroup).one()
|
||||
assert doc.title == 'example word.doc'
|
||||
assert doc.title == 'example word doc'
|
||||
assert doc.a_text == 'example word doc'
|
||||
if IS_MACOS:
|
||||
assert doc.d_text == 'Example Word Document\nSecond page'
|
||||
|
|
@ -962,8 +962,9 @@ async def test_docx_indexer(async_client, test_session, example_docx, refresh_fi
|
|||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
doc = test_session.query(FileGroup).one()
|
||||
assert doc.title == 'example word.docx'
|
||||
assert doc.a_text == 'example word docx'
|
||||
# The docx file has a "Word Document" title in its core properties.
|
||||
assert doc.title == 'Word Document'
|
||||
assert doc.a_text == 'Word Document'
|
||||
assert doc.d_text == 'Example Word Document Second page'
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -132,5 +132,3 @@ def test_file_group_update_wrolpi_json_noop_no_metadata_json(test_session, make_
|
|||
fg.update_wrolpi_json({'key': 'value'})
|
||||
assert not (get_media_directory() / 'video.info.json').exists()
|
||||
assert not (get_media_directory() / 'video.readability.json').exists()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
"""Test for pdf_modeler loop to ensure it processes more than one batch.
|
||||
|
||||
This test ensures the pdf_modeler can process more than PDF_PROCESSING_LIMIT (10)
|
||||
files. If the loop logic has an off-by-one error (like using enumerate() with wrong
|
||||
comparison), the modeler would only process one batch and exit early.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.files.pdfs import pdf_modeler, PDF_PROCESSING_LIMIT
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory):
|
||||
"""
|
||||
Test that pdf_modeler processes MORE than PDF_PROCESSING_LIMIT (10) files.
|
||||
|
||||
This test creates 15 PDF files and verifies that pdf_modeler
|
||||
processes all of them, not just the first batch of 10.
|
||||
|
||||
This catches off-by-one bugs in the loop logic (e.g., using enumerate()
|
||||
which is 0-indexed but comparing against the limit incorrectly).
|
||||
"""
|
||||
pdf_dir = test_directory / 'pdfs'
|
||||
pdf_dir.mkdir(parents=True)
|
||||
|
||||
# Create more PDFs than the batch limit
|
||||
num_pdfs = PDF_PROCESSING_LIMIT + 5 # 15 PDFs
|
||||
pdf_paths = []
|
||||
|
||||
for i in range(num_pdfs):
|
||||
pdf_path = pdf_dir / f'test_pdf_{i:03d}.pdf'
|
||||
shutil.copy(PROJECT_DIR / 'test/pdf example.pdf', pdf_path)
|
||||
pdf_paths.append(pdf_path)
|
||||
|
||||
# Create FileGroups for each PDF file (simulating what refresh does)
|
||||
for pdf_path in pdf_paths:
|
||||
fg = FileGroup.from_paths(test_session, pdf_path)
|
||||
assert fg.mimetype == 'application/pdf'
|
||||
|
||||
test_session.commit()
|
||||
|
||||
# Verify we have the expected number of unindexed PDF FileGroups
|
||||
unindexed_count = test_session.query(FileGroup).filter(
|
||||
FileGroup.indexed != True,
|
||||
FileGroup.mimetype == 'application/pdf',
|
||||
).count()
|
||||
assert unindexed_count == num_pdfs, f"Expected {num_pdfs} unindexed PDF files, got {unindexed_count}"
|
||||
|
||||
# Run the pdf_modeler
|
||||
await pdf_modeler()
|
||||
|
||||
# Count how many FileGroups are now indexed as PDFs
|
||||
indexed_pdf_count = test_session.query(FileGroup).filter(
|
||||
FileGroup.indexed == True,
|
||||
FileGroup.model == 'pdf',
|
||||
).count()
|
||||
|
||||
# All PDFs should be indexed
|
||||
assert indexed_pdf_count == num_pdfs, \
|
||||
f"pdf_modeler should process ALL {num_pdfs} files, but only processed {indexed_pdf_count}. " \
|
||||
f"This may be an off-by-one bug in the loop logic!"
|
||||
|
||||
# Also verify no FileGroups remain unindexed
|
||||
still_unindexed = test_session.query(FileGroup).filter(
|
||||
FileGroup.indexed != True,
|
||||
FileGroup.mimetype == 'application/pdf',
|
||||
).count()
|
||||
assert still_unindexed == 0, \
|
||||
f"All PDF FileGroups should be indexed, but {still_unindexed} remain unindexed"
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files import pdfs
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_title(async_client, test_session, example_pdf, refresh_files):
|
||||
"""PDF modeler extracts a PDFs title and contents."""
|
||||
await refresh_files()
|
||||
|
||||
with mock.patch('wrolpi.files.pdfs.get_pdf_metadata') as mock_get_pdf_metadata:
|
||||
mock_get_pdf_metadata.side_effect = Exception('Should not index twice')
|
||||
await refresh_files()
|
||||
|
||||
pdf: FileGroup = test_session.query(FileGroup).one()
|
||||
assert pdf.indexed is True
|
||||
assert pdf.title == pdf.a_text == 'WROLPi Test PDF'
|
||||
assert pdf.b_text == 'roland'
|
||||
assert pdf.c_text == 'pdf example pdf'
|
||||
assert pdf.d_text and pdf.d_text.startswith('Page one\n') and len(pdf.d_text) == 467
|
||||
# Metadata within the PDF.
|
||||
assert pdf.author
|
||||
assert pdf.published_datetime
|
||||
# The date the file was modified.
|
||||
assert pdf.modification_datetime
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_modeler(test_session, example_pdf):
|
||||
"""PDFs can be indexed by PDFIndexer."""
|
||||
files_lib._upsert_files([example_pdf], now())
|
||||
|
||||
await pdfs.pdf_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
a_text, b_text, c_text, d_text = file_group.a_text, file_group.b_text, file_group.c_text, file_group.d_text
|
||||
|
||||
# The title extracted from the PDF.
|
||||
assert a_text == 'WROLPi Test PDF'
|
||||
# The author.
|
||||
assert b_text == 'roland'
|
||||
# The parsed file name.
|
||||
assert c_text == 'pdf example pdf'
|
||||
# All pages are extracted. Text is formatted to fit on vertical screen.
|
||||
assert d_text == 'Page one\n' \
|
||||
'Page two\n' \
|
||||
'Lorem ipsum dolor sit amet,\n' \
|
||||
'consectetur adipiscing elit, sed do\n' \
|
||||
'eiusmod tempor incididunt ut labore et\n' \
|
||||
'\n' \
|
||||
'dolore magna aliqua. Ut enim ad minim\n' \
|
||||
'veniam, quis nostrud exercitation\n' \
|
||||
'ullamco laboris nisi ut \n' \
|
||||
'aliquip ex ea commodo consequat. Duis\n' \
|
||||
'aute irure dolor in reprehenderit in\n' \
|
||||
'voluptate velit esse cillum \n' \
|
||||
'dolore eu fugiat nulla pariatur.\n' \
|
||||
'Excepteur sint occaecat cupidatat non\n' \
|
||||
'proident, sunt in culpa qui officia \n' \
|
||||
'deserunt mollit anim id est laborum.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_indexer_max_size(test_session, example_pdf):
|
||||
"""The contents of a large PDF are not indexed."""
|
||||
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
|
||||
|
||||
files_lib._upsert_files([example_pdf], now())
|
||||
|
||||
await pdfs.pdf_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
assert file_group.d_text is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_poster(test_session, example_pdf):
|
||||
"""PDF modeler finds an image with the same stem and uses it as the poster."""
|
||||
# Create an image with the same stem as the PDF.
|
||||
poster_path = example_pdf.with_suffix('.jpg')
|
||||
Image.new('RGB', (25, 25), color='grey').save(poster_path)
|
||||
|
||||
files_lib._upsert_files([example_pdf, poster_path], now())
|
||||
|
||||
await pdfs.pdf_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
assert file_group.indexed is True
|
||||
assert file_group.data is not None
|
||||
# data['poster_path'] stores filename only (for fast moves/renames).
|
||||
assert file_group.data.get('poster_path') == poster_path.name
|
||||
|
|
@ -473,7 +473,8 @@ async def test_file_worker_status_during_refresh(async_client, test_session, tes
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_status_resets_after_refresh(async_client, test_session, test_directory, make_files_structure):
|
||||
async def test_file_worker_status_resets_after_refresh(async_client, test_session, test_directory,
|
||||
make_files_structure):
|
||||
"""FileWorker resets status to idle after refresh completes."""
|
||||
make_files_structure(['docs/file1.txt'])
|
||||
|
||||
|
|
@ -509,7 +510,7 @@ async def test_file_worker_count_only_resets_status(async_client, test_session,
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_refresh_inserts_new_files(async_client, test_session, test_directory, make_files_structure,
|
||||
video_file_factory):
|
||||
video_file_factory):
|
||||
"""FileWorker inserts new FileGroups during refresh."""
|
||||
# Create files on disk but not in DB
|
||||
# Note: txt/json files without recognizable primary types (video, epub, etc.)
|
||||
|
|
@ -898,7 +899,8 @@ async def test_handle_refresh_runs_modelers(async_client, test_session, test_dir
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_refresh_status_includes_indexing(async_client, test_session, test_directory, make_files_structure):
|
||||
async def test_handle_refresh_status_includes_indexing(async_client, test_session, test_directory,
|
||||
make_files_structure):
|
||||
"""FileWorker.handle_refresh should update status during indexing phase."""
|
||||
# Create files
|
||||
make_files_structure(['file1.txt', 'file2.txt'])
|
||||
|
|
@ -926,7 +928,8 @@ async def test_handle_refresh_status_includes_indexing(async_client, test_sessio
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_complete_flag_set_on_full_refresh(async_client, test_session, test_directory, make_files_structure, flags_lock):
|
||||
async def test_refresh_complete_flag_set_on_full_refresh(async_client, test_session, test_directory,
|
||||
make_files_structure, flags_lock):
|
||||
"""refresh_complete flag is set when refreshing the entire media directory."""
|
||||
from wrolpi import flags
|
||||
|
||||
|
|
@ -946,7 +949,8 @@ async def test_refresh_complete_flag_set_on_full_refresh(async_client, test_sess
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_complete_flag_not_set_on_partial_refresh(async_client, test_session, test_directory, make_files_structure, flags_lock):
|
||||
async def test_refresh_complete_flag_not_set_on_partial_refresh(async_client, test_session, test_directory,
|
||||
make_files_structure, flags_lock):
|
||||
"""refresh_complete flag is NOT set when refreshing only a subdirectory."""
|
||||
from wrolpi import flags
|
||||
|
||||
|
|
@ -1251,7 +1255,7 @@ async def test_file_worker_refresh_file_skips_count_phase(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_refresh_returns_job_id(async_client, test_session, test_directory, make_files_structure,
|
||||
await_background_tasks):
|
||||
await_background_tasks):
|
||||
"""queue_refresh returns a job_id that can be tracked to completion."""
|
||||
make_files_structure(['docs/file1.txt'])
|
||||
|
||||
|
|
@ -1270,7 +1274,7 @@ async def test_queue_refresh_returns_job_id(async_client, test_session, test_dir
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_job(async_client, test_session, test_directory, make_files_structure,
|
||||
await_background_tasks):
|
||||
await_background_tasks):
|
||||
"""wait_for_job waits for a job to complete."""
|
||||
make_files_structure(['docs/file1.txt'])
|
||||
|
||||
|
|
@ -1291,7 +1295,7 @@ async def test_wait_for_job(async_client, test_session, test_directory, make_fil
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_counting_flag_is_set(async_client, test_session, test_directory, make_files_structure,
|
||||
flags_lock):
|
||||
flags_lock):
|
||||
"""file_worker_counting flag is set during counting phase."""
|
||||
from wrolpi import flags
|
||||
|
||||
|
|
@ -1320,7 +1324,7 @@ async def test_file_worker_counting_flag_is_set(async_client, test_session, test
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_discovery_flag_is_set(async_client, test_session, test_directory, make_files_structure,
|
||||
flags_lock):
|
||||
flags_lock):
|
||||
"""file_worker_discovery flag is set during discovery phases (comparing, upserting, deleting)."""
|
||||
from wrolpi import flags
|
||||
|
||||
|
|
@ -1362,7 +1366,7 @@ async def test_upsert_files_progress_callback(test_session, test_directory, make
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upsert_file_groups_tracks_actual_db_progress(async_client, test_session, test_directory,
|
||||
make_files_structure):
|
||||
make_files_structure):
|
||||
"""_upsert_file_groups tracks progress of actual DB operations, not just diff collection."""
|
||||
make_files_structure(['docs/file1.txt', 'docs/file2.txt', 'docs/file3.txt'])
|
||||
|
||||
|
|
|
|||
|
|
@ -1392,7 +1392,7 @@ class FileWorker:
|
|||
logger.info(f'Deleted {deleted_count} FileGroups ({len(all_urls)} URLs added to skip list)')
|
||||
|
||||
def _cleanup_modified_models(self, diffs: list[FileGroupDiff]):
|
||||
"""Delete Videos/Archives/EBooks for modified FileGroups where primary file was removed.
|
||||
"""Delete Videos/Archives/Docs for modified FileGroups where primary file was removed.
|
||||
|
||||
Optimized: uses a single UNION ALL query to fetch all model types at once,
|
||||
reducing 3 SELECT queries to 1.
|
||||
|
|
@ -1421,16 +1421,16 @@ class FileWorker:
|
|||
JOIN file_group fg ON a.file_group_id = fg.id
|
||||
WHERE fg.id = ANY (%s)
|
||||
UNION ALL
|
||||
SELECT 'ebook', e.id, fg.id, fg.primary_path
|
||||
FROM ebook e
|
||||
JOIN file_group fg ON e.file_group_id = fg.id
|
||||
SELECT 'doc', d.id, fg.id, fg.primary_path
|
||||
FROM doc d
|
||||
JOIN file_group fg ON d.file_group_id = fg.id
|
||||
WHERE fg.id = ANY (%s)
|
||||
''', (file_group_ids, file_group_ids, file_group_ids))
|
||||
|
||||
# Group results by model type
|
||||
video_ids_to_delete = []
|
||||
archive_ids_to_delete = []
|
||||
ebook_ids_to_delete = []
|
||||
doc_ids_to_delete = []
|
||||
|
||||
for model_type, model_id, fg_id, primary_path in curs.fetchall():
|
||||
basename = pathlib.Path(primary_path).name
|
||||
|
|
@ -1439,8 +1439,8 @@ class FileWorker:
|
|||
video_ids_to_delete.append(model_id)
|
||||
elif model_type == 'archive':
|
||||
archive_ids_to_delete.append(model_id)
|
||||
elif model_type == 'ebook':
|
||||
ebook_ids_to_delete.append(model_id)
|
||||
elif model_type == 'doc':
|
||||
doc_ids_to_delete.append(model_id)
|
||||
|
||||
# Batch delete each model type
|
||||
if video_ids_to_delete:
|
||||
|
|
@ -1461,14 +1461,14 @@ class FileWorker:
|
|||
''', (archive_ids_to_delete,))
|
||||
curs.execute('DELETE FROM archive WHERE id = ANY(%s)', (archive_ids_to_delete,))
|
||||
|
||||
if ebook_ids_to_delete:
|
||||
logger.info(f'Deleting {len(ebook_ids_to_delete)} EBooks whose primary file was removed')
|
||||
if doc_ids_to_delete:
|
||||
logger.info(f'Deleting {len(doc_ids_to_delete)} Docs whose primary file was removed')
|
||||
curs.execute('''
|
||||
UPDATE file_group
|
||||
SET model = NULL
|
||||
WHERE id IN (SELECT file_group_id FROM ebook WHERE id = ANY (%s))
|
||||
''', (ebook_ids_to_delete,))
|
||||
curs.execute('DELETE FROM ebook WHERE id = ANY(%s)', (ebook_ids_to_delete,))
|
||||
WHERE id IN (SELECT file_group_id FROM doc WHERE id = ANY (%s))
|
||||
''', (doc_ids_to_delete,))
|
||||
curs.execute('DELETE FROM doc WHERE id = ANY(%s)', (doc_ids_to_delete,))
|
||||
|
||||
def _validate_move_paths(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from vininfo import Vin
|
|||
from vininfo.details._base import VinDetails
|
||||
|
||||
from modules.archive.api import archive_bp
|
||||
from modules.docs.api import docs_bp
|
||||
from modules.inventory import inventory_bp
|
||||
from modules.map.api import map_bp
|
||||
from modules.otp.api import otp_bp
|
||||
|
|
@ -49,6 +50,7 @@ api_app.blueprint(api_bp)
|
|||
api_app.blueprint(archive_bp)
|
||||
api_app.blueprint(collection_bp) # Unified collection endpoints
|
||||
api_app.blueprint(config_bp)
|
||||
api_app.blueprint(docs_bp)
|
||||
api_app.blueprint(files_bp)
|
||||
api_app.blueprint(inventory_bp)
|
||||
api_app.blueprint(map_bp)
|
||||
|
|
@ -710,16 +712,21 @@ async def post_upgrade_start(_: Request):
|
|||
async def post_search_suggestions(request: Request, body: schema.SearchSuggestionsRequest):
|
||||
from modules.videos.channel.lib import search_channels_by_name
|
||||
from modules.archive.lib import search_domains_by_name
|
||||
from modules.docs.lib import search_authors_by_name, search_subjects_by_name
|
||||
|
||||
session = request.ctx.session
|
||||
channels, domains = await asyncio.gather(
|
||||
channels, domains, authors, subjects = await asyncio.gather(
|
||||
search_channels_by_name(session, body.search_str, order_by_video_count=body.order_by_video_count),
|
||||
search_domains_by_name(session, body.search_str),
|
||||
search_authors_by_name(session, body.search_str),
|
||||
search_subjects_by_name(session, body.search_str),
|
||||
)
|
||||
|
||||
ret = dict(
|
||||
channels=channels,
|
||||
domains=domains,
|
||||
authors=authors,
|
||||
subjects=subjects,
|
||||
)
|
||||
return json_response(ret)
|
||||
|
||||
|
|
|
|||
|
|
@ -152,5 +152,6 @@ async def await_switches(timeout: int = 8):
|
|||
# All switches handled.
|
||||
break
|
||||
else:
|
||||
logger.error(f'await_switches: TIMEOUT after {count} iterations, switches={list(api_app.shared_ctx.switches.keys())}, changed={api_app.shared_ctx.switches_changed.is_set()}')
|
||||
logger.error(
|
||||
f'await_switches: TIMEOUT after {count} iterations, switches={list(api_app.shared_ctx.switches.keys())}, changed={api_app.shared_ctx.switches_changed.is_set()}')
|
||||
raise RuntimeError('Timed out waiting for switches. Did you remember to use the `await_switches` fixture?')
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
import contextlib
|
||||
import pathlib
|
||||
from dataclasses import dataclass, field
|
||||
|
|
@ -517,7 +516,8 @@ class TagsConfig(ConfigFile):
|
|||
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
|
||||
backup_file = self._get_backup_file(backup_date)
|
||||
backup_data = self.read_config_file(backup_file)
|
||||
current_data = self.read_config_file() if self.get_file().is_file() else dict(tags={}, tag_files=[], tag_zims=[])
|
||||
current_data = self.read_config_file() if self.get_file().is_file() else dict(tags={}, tag_files=[],
|
||||
tag_zims=[])
|
||||
|
||||
backup_tags = backup_data.get('tags', {})
|
||||
current_tags = current_data.get('tags', {})
|
||||
|
|
@ -586,7 +586,8 @@ class TagsConfig(ConfigFile):
|
|||
shutil.copy2(backup_file, config_file)
|
||||
elif mode == 'merge':
|
||||
backup_data = self.read_config_file(backup_file)
|
||||
current_data = self.read_config_file() if config_file.is_file() else dict(tags={}, tag_files=[], tag_zims=[], version=0)
|
||||
current_data = self.read_config_file() if config_file.is_file() else dict(tags={}, tag_files=[],
|
||||
tag_zims=[], version=0)
|
||||
|
||||
# Merge tags: add missing tags only
|
||||
merged_tags = dict(current_data.get('tags', {}))
|
||||
|
|
@ -791,10 +792,10 @@ def get_tags() -> List[dict]:
|
|||
SELECT t.id,
|
||||
t.name,
|
||||
t.color,
|
||||
(SELECT COUNT(*) FROM tag_file WHERE tag_id = t.id) AS file_group_count,
|
||||
(SELECT COUNT(*) FROM tag_zim WHERE tag_id = t.id) AS zim_entry_count,
|
||||
(SELECT COUNT(*) FROM tag_file WHERE tag_id = t.id) AS file_group_count,
|
||||
(SELECT COUNT(*) FROM tag_zim WHERE tag_id = t.id) AS zim_entry_count,
|
||||
(SELECT COUNT(*) FROM collection WHERE tag_id = t.id AND kind = 'channel') AS channel_count,
|
||||
(SELECT COUNT(*) FROM collection WHERE tag_id = t.id AND kind = 'domain') AS domain_count
|
||||
(SELECT COUNT(*) FROM collection WHERE tag_id = t.id AND kind = 'domain') AS domain_count
|
||||
FROM tag t
|
||||
GROUP BY t.id, t.name, t.color
|
||||
ORDER BY t.name
|
||||
|
|
@ -806,15 +807,18 @@ def get_tags() -> List[dict]:
|
|||
def get_recent_tags(limit: int = 5) -> List[str]:
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('''
|
||||
SELECT t.name FROM (
|
||||
SELECT tag_id, MAX(created_at) AS latest FROM (
|
||||
SELECT tag_id, created_at FROM tag_file
|
||||
UNION ALL
|
||||
SELECT tag_id, created_at FROM tag_zim
|
||||
) AS combined
|
||||
GROUP BY tag_id ORDER BY latest DESC LIMIT %(limit)s
|
||||
) AS recent
|
||||
JOIN tag t ON t.id = recent.tag_id ORDER BY recent.latest DESC
|
||||
SELECT t.name
|
||||
FROM (SELECT tag_id, MAX(created_at) AS latest
|
||||
FROM (SELECT tag_id, created_at
|
||||
FROM tag_file
|
||||
UNION ALL
|
||||
SELECT tag_id, created_at
|
||||
FROM tag_zim) AS combined
|
||||
GROUP BY tag_id
|
||||
ORDER BY latest DESC
|
||||
LIMIT %(limit)s) AS recent
|
||||
JOIN tag t ON t.id = recent.tag_id
|
||||
ORDER BY recent.latest DESC
|
||||
''', dict(limit=limit))
|
||||
return [row['name'] for row in curs.fetchall()]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
version: 0
|
||||
skip_urls: []
|
||||
skip_urls: [ ]
|
||||
downloads:
|
||||
- url: 'https://example.com/channel'
|
||||
downloader: video
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
version: 0
|
||||
skip_urls: []
|
||||
skip_urls: [ ]
|
||||
downloads:
|
||||
- url: 'https://example.com/minimal'
|
||||
downloader: video
|
||||
|
|
@ -8,4 +8,4 @@ downloads:
|
|||
destination: null
|
||||
sub_downloader: null
|
||||
settings: null
|
||||
tag_names: []
|
||||
tag_names: [ ]
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ class TestTagsBackupImport:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_merge(self, test_directory, test_tags_config, test_session, async_client):
|
||||
from wrolpi.tags import get_tags_config, Tag
|
||||
from wrolpi.tags import get_tags_config
|
||||
config = get_tags_config()
|
||||
|
||||
# Write current config with one tag
|
||||
|
|
@ -148,7 +148,7 @@ class TestTagsBackupImport:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_overwrite(self, test_directory, test_tags_config, test_session, async_client):
|
||||
from wrolpi.tags import get_tags_config, Tag
|
||||
from wrolpi.tags import get_tags_config
|
||||
config = get_tags_config()
|
||||
|
||||
# Write current config with two tags
|
||||
|
|
@ -641,7 +641,6 @@ class TestImportBackupPreservesCurrentConfig:
|
|||
assert preserved['version'] == 5
|
||||
|
||||
|
||||
|
||||
class TestConfigStatusIncludesBackupInfo:
|
||||
|
||||
def test_db_config_has_backup_info(self, test_directory, test_tags_config):
|
||||
|
|
@ -678,7 +677,6 @@ class TestNotImplementedForNonDBConfigs:
|
|||
class TestBackupAPIEndpoints:
|
||||
|
||||
async def test_get_backups(self, async_client, test_directory, test_tags_config):
|
||||
from wrolpi.tags import get_tags_config
|
||||
backup_dir = test_directory / 'config' / 'backup'
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
(backup_dir / 'tags-20260301.yaml').write_text('tags: {}\nversion: 1')
|
||||
|
|
@ -881,7 +879,8 @@ class TestPreviewWithRealisticData:
|
|||
current = dict(
|
||||
downloads=[
|
||||
dict(url='https://example.com', downloader='video', frequency=None, status='failed'),
|
||||
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000, status='complete'),
|
||||
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000,
|
||||
status='complete'),
|
||||
],
|
||||
skip_urls=['https://skip1.com'],
|
||||
version=260,
|
||||
|
|
@ -892,7 +891,8 @@ class TestPreviewWithRealisticData:
|
|||
backup = dict(
|
||||
downloads=[
|
||||
dict(url='https://example.com', downloader='video', frequency=None, status='failed'),
|
||||
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000, status='complete'),
|
||||
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000,
|
||||
status='complete'),
|
||||
dict(url='https://youtube.com/watch?v=TEST', downloader='video', frequency=None, status='new'),
|
||||
],
|
||||
skip_urls=['https://skip1.com'],
|
||||
|
|
@ -914,7 +914,8 @@ class TestPreviewWithRealisticData:
|
|||
current = dict(
|
||||
downloads=[
|
||||
dict(url='https://example.com', downloader='video', frequency=None, status='failed'),
|
||||
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000, status='complete'),
|
||||
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000,
|
||||
status='complete'),
|
||||
],
|
||||
skip_urls=[],
|
||||
version=260,
|
||||
|
|
@ -924,7 +925,8 @@ class TestPreviewWithRealisticData:
|
|||
backup = dict(
|
||||
downloads=[
|
||||
dict(url='https://example.com', downloader='video', frequency=None, status='failed'),
|
||||
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000, status='complete'),
|
||||
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000,
|
||||
status='complete'),
|
||||
dict(url='https://youtube.com/watch?v=TEST', downloader='video', frequency=None, status='new'),
|
||||
],
|
||||
skip_urls=[],
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import asyncio
|
|||
import json
|
||||
import pathlib
|
||||
from abc import ABC
|
||||
from datetime import datetime, time, timezone, timedelta
|
||||
from datetime import datetime, time, timezone
|
||||
from http import HTTPStatus
|
||||
from itertools import zip_longest
|
||||
from unittest import mock
|
||||
|
|
@ -16,7 +16,7 @@ from wrolpi.dates import Seconds
|
|||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
|
||||
get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \
|
||||
set_download_progress, clear_download_progress, make_progress_callback, download_manager
|
||||
set_download_progress, clear_download_progress, make_progress_callback
|
||||
from wrolpi.errors import InvalidDownload, WROLModeEnabled
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
|
@ -1196,7 +1196,6 @@ def test_is_within_download_window_uses_local_time(test_download_manager):
|
|||
|
||||
def test_get_local_time_with_configured_timezone(test_download_manager):
|
||||
"""When config.timezone is set, _get_local_time() converts to that timezone."""
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
config = get_wrolpi_config()
|
||||
config.timezone = 'America/Denver'
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ class TestDownloadManagerConfigEdgeCases:
|
|||
assert downloads[0].url == 'https://example.com/new'
|
||||
|
||||
async def test_import_config_with_missing_optional_fields(self, test_session, test_directory,
|
||||
test_download_manager_config, async_client):
|
||||
test_download_manager_config, async_client):
|
||||
"""
|
||||
Config files from older versions or manual edits may be missing optional fields.
|
||||
Import should succeed with None values for missing fields.
|
||||
|
|
@ -331,7 +331,7 @@ class TestDownloadManagerConfigEdgeCases:
|
|||
assert downloads[0].next_download is None
|
||||
|
||||
async def test_update_existing_download_with_missing_optional_fields(self, test_session, test_directory,
|
||||
test_download_manager_config, async_client):
|
||||
test_download_manager_config, async_client):
|
||||
"""
|
||||
When updating an existing download from config, missing optional fields should not cause errors.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from wrolpi.api_utils import json_error_handler
|
|||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.downloader import Download, get_download_manager_config
|
||||
from wrolpi.errors import ValidationError, SearchEmpty
|
||||
from wrolpi.test.common import skip_circleci, assert_dict_contains
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -248,7 +248,8 @@ async def test_rss_downloader_normalizes_youtube_shorts_urls(test_session, test_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_session, test_download_manager, await_switches):
|
||||
async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_session, test_download_manager,
|
||||
await_switches):
|
||||
"""RSS downloader should normalize YouTube Shorts URLs before checking the skip list.
|
||||
|
||||
If a video was skipped with a normalized URL, the RSS feed returning a shorts URL should
|
||||
|
|
|
|||
|
|
@ -548,7 +548,7 @@ async def test_tags_directory_file_replaced(test_session, test_directory, tag_fa
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_directory_partial_file_deleted(test_session, test_directory, tag_factory, video_bytes,
|
||||
image_bytes_factory, await_switches):
|
||||
image_bytes_factory, await_switches):
|
||||
"""When one file in a FileGroup is deleted, its hardlink should be removed."""
|
||||
from wrolpi import tags as tags_module
|
||||
|
||||
|
|
@ -584,7 +584,7 @@ async def test_tags_directory_partial_file_deleted(test_session, test_directory,
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_directory_all_files_deleted(test_session, test_directory, tag_factory, video_bytes,
|
||||
image_bytes_factory, await_switches):
|
||||
image_bytes_factory, await_switches):
|
||||
"""When ALL files in a FileGroup are deleted, hardlinks should be preserved."""
|
||||
from wrolpi import tags as tags_module
|
||||
|
||||
|
|
@ -728,8 +728,6 @@ async def test_recent_tags_api(async_client, test_session, make_files_structure,
|
|||
assert 'alpha' in body['tag_names']
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_overlapping_tags_multiple(test_session, make_files_structure, tag_factory, video_bytes,
|
||||
refresh_files):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import pathlib
|
|||
import pytest
|
||||
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.common import get_media_directory
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue