Add Docs module for searching and organizing PDFs and EPUBs

Create a new Docs module that consolidates PDF and eBook handling into a
unified system with subjects and authors support. This replaces the
separate PDF/eBook modelers with a single Doc model backed by an
alembic migration (rename ebook to doc).

Key changes:
- New modules/docs package with models, API, extractors, and lib
- Subject and author metadata extraction with case-insensitive
  normalization for cleaner PDF metadata
- Frontend Docs pages with AuthorsPage, SubjectsPage using
  CollectionTable, and doc card deletion
- Global search integration for subjects and authors
- Generic collection item count support
- Frontend file card metadata adjusted to match collection sort
- Bump yt-dlp version
This commit is contained in:
Roland 2026-03-17 18:35:57 -06:00
parent 4a21499bdd
commit aefc7b1def
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
90 changed files with 4484 additions and 1189 deletions

View file

@ -18,7 +18,7 @@ fileConfig(config.config_file_name)
from wrolpi.common import Base
# Import all models so they are registered with the Base metadata
from wrolpi.files.models import FileGroup, Directory # noqa: F401
from wrolpi.files.ebooks import EBook # noqa: F401
from modules.docs.models import Doc # noqa: F401
from wrolpi.tags import Tag # noqa: F401
from wrolpi.flags import WROLPiFlag # noqa: F401
from wrolpi.downloader import Download # noqa: F401

View file

@ -0,0 +1,172 @@
"""Rename ebook table to doc and add new columns.
Revision ID: a1b2c3d4e5f6
Revises: d371333159ab
Create Date: 2026-03-15 15:30:00.000000
"""
import os
from alembic import op
import sqlalchemy as sa
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = 'e7f8a9b0c1d2'
down_revision = 'd371333159ab'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
BATCH_SIZE = 1000
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
if not DOCKERIZED:
bind.execute(sa.text("SET ROLE 'wrolpi'"))
# Rename ebook table to doc.
print("Migrating ebook -> doc...")
op.rename_table('ebook', 'doc')
# Rename the index.
op.execute('ALTER INDEX IF EXISTS ebook_size_idx RENAME TO doc_size_idx')
op.execute('ALTER INDEX IF EXISTS ebook_file_group_id_key RENAME TO doc_file_group_id_key')
op.execute('ALTER INDEX IF EXISTS ebook_pkey RENAME TO doc_pkey')
# Rename the sequence.
op.execute('ALTER SEQUENCE IF EXISTS ebook_id_seq RENAME TO doc_id_seq')
# Add new columns.
op.add_column('doc', sa.Column('publisher', sa.String(), nullable=True))
op.add_column('doc', sa.Column('language', sa.String(), nullable=True))
op.add_column('doc', sa.Column('page_count', sa.Integer(), nullable=True))
op.add_column('doc', sa.Column('subject', sa.String(), nullable=True))
op.add_column('doc', sa.Column('description', sa.Text(), nullable=True))
# Widen size from Integer to BigInteger.
op.alter_column('doc', 'size', type_=sa.BigInteger(), existing_type=sa.Integer())
# Update file_group.model: 'ebook' -> 'doc'.
# Mark all touched file_groups as not indexed so metadata is re-extracted on next refresh.
total = 0
while True:
result = bind.execute(sa.text(
"UPDATE file_group SET model = 'doc', indexed = false"
" WHERE id IN (SELECT id FROM file_group WHERE model = 'ebook' LIMIT :batch_size)"
), {'batch_size': BATCH_SIZE})
if result.rowcount == 0:
break
total += result.rowcount
print(f" Updated {total} ebook file_groups -> doc...")
session.commit()
print(f" Updated {total} ebook file_groups -> doc (done)")
# Update file_group.model: 'pdf' -> 'doc'.
total = 0
while True:
result = bind.execute(sa.text(
"UPDATE file_group SET model = 'doc', indexed = false"
" WHERE id IN (SELECT id FROM file_group WHERE model = 'pdf' LIMIT :batch_size)"
), {'batch_size': BATCH_SIZE})
if result.rowcount == 0:
break
total += result.rowcount
print(f" Updated {total} pdf file_groups -> doc...")
session.commit()
print(f" Updated {total} pdf file_groups -> doc (done)")
# Create Doc records for existing PDFs that had no model table.
# Metadata will be extracted when the doc_modeler re-indexes these files.
total = 0
while True:
result = bind.execute(sa.text("""
INSERT INTO doc (file_group_id)
SELECT fg.id
FROM file_group fg
LEFT JOIN doc d ON d.file_group_id = fg.id
WHERE fg.mimetype = 'application/pdf'
AND d.id IS NULL
AND fg.primary_path IS NOT NULL
LIMIT :batch_size
"""), {'batch_size': BATCH_SIZE})
if result.rowcount == 0:
break
total += result.rowcount
print(f" Created {total} new doc records for existing PDFs...")
session.commit()
print(f" Created {total} new doc records for existing PDFs (done)")
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
if not DOCKERIZED:
bind.execute(sa.text("SET ROLE 'wrolpi'"))
# Delete doc records that were PDFs (have no ebook equivalent).
total = 0
while True:
result = bind.execute(sa.text("""
DELETE FROM doc
WHERE id IN (
SELECT d.id FROM doc d
JOIN file_group fg ON fg.id = d.file_group_id
WHERE fg.mimetype = 'application/pdf'
LIMIT :batch_size
)
"""), {'batch_size': BATCH_SIZE})
if result.rowcount == 0:
break
total += result.rowcount
print(f" Deleted {total} PDF doc records...")
session.commit()
print(f" Deleted {total} PDF doc records (done)")
# Revert file_group.model and re-mark as not indexed.
for old_model, mimetype_clause in [
('ebook', "mimetype LIKE 'application/epub%'"),
('ebook', "mimetype = 'application/x-mobipocket-ebook'"),
('pdf', "mimetype = 'application/pdf'"),
]:
total = 0
while True:
result = bind.execute(sa.text(f"""
UPDATE file_group SET model = :old_model, indexed = false
WHERE id IN (
SELECT id FROM file_group
WHERE model = 'doc' AND {mimetype_clause}
LIMIT :batch_size
)
"""), {'old_model': old_model, 'batch_size': BATCH_SIZE})
if result.rowcount == 0:
break
total += result.rowcount
print(f" Reverted {total} file_groups -> {old_model} ({mimetype_clause})...")
session.commit()
print(f" Reverted {total} file_groups -> {old_model} ({mimetype_clause}) (done)")
# Revert size column type.
op.alter_column('doc', 'size', type_=sa.Integer(), existing_type=sa.BigInteger())
# Drop new columns.
op.drop_column('doc', 'description')
op.drop_column('doc', 'subject')
op.drop_column('doc', 'page_count')
op.drop_column('doc', 'language')
op.drop_column('doc', 'publisher')
# Rename sequences back.
op.execute('ALTER SEQUENCE IF EXISTS doc_id_seq RENAME TO ebook_id_seq')
# Rename indexes back.
op.execute('ALTER INDEX IF EXISTS doc_pkey RENAME TO ebook_pkey')
op.execute('ALTER INDEX IF EXISTS doc_file_group_id_key RENAME TO ebook_file_group_id_key')
op.execute('ALTER INDEX IF EXISTS doc_size_idx RENAME TO ebook_size_idx')
# Rename table back.
op.rename_table('doc', 'ebook')

86
app/package-lock.json generated
View file

@ -97,6 +97,7 @@
"version": "7.20.12",
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz",
"integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==",
"peer": true,
"dependencies": {
"@ampproject/remapping": "^2.1.0",
"@babel/code-frame": "^7.18.6",
@ -903,6 +904,7 @@
"version": "7.18.6",
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz",
"integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==",
"peer": true,
"dependencies": {
"@babel/helper-plugin-utils": "^7.18.6"
},
@ -1464,6 +1466,7 @@
"version": "7.20.13",
"resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.20.13.tgz",
"integrity": "sha512-MmTZx/bkUrfJhhYAYt3Urjm+h8DQGrPrnKQ94jLo7NLuOU+T89a7IByhKmrb8SKhrIYIQ0FN0CHMbnFRen4qNw==",
"peer": true,
"dependencies": {
"@babel/helper-annotate-as-pure": "^7.18.6",
"@babel/helper-module-imports": "^7.18.6",
@ -3283,6 +3286,7 @@
"version": "2.11.6",
"resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz",
"integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==",
"peer": true,
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/popperjs"
@ -4066,6 +4070,7 @@
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.27.tgz",
"integrity": "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/prop-types": "*",
"csstype": "^3.2.2"
@ -4076,6 +4081,7 @@
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz",
"integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==",
"license": "MIT",
"peer": true,
"peerDependencies": {
"@types/react": "^18.0.0"
}
@ -4190,6 +4196,7 @@
"version": "5.50.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.50.0.tgz",
"integrity": "sha512-vwksQWSFZiUhgq3Kv7o1Jcj0DUNylwnIlGvKvLLYsq8pAWha6/WCnXUeaSoNNha/K7QSf2+jvmkxggC1u3pIwQ==",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "5.50.0",
"@typescript-eslint/type-utils": "5.50.0",
@ -4241,6 +4248,7 @@
"version": "5.50.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.50.0.tgz",
"integrity": "sha512-KCcSyNaogUDftK2G9RXfQyOCt51uB5yqC6pkUYqhYh8Kgt+DwR5M0EwEAxGPy/+DH6hnmKeGsNhiZRQxjH71uQ==",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "5.50.0",
"@typescript-eslint/types": "5.50.0",
@ -4571,6 +4579,7 @@
"version": "8.15.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@ -4693,6 +4702,7 @@
"version": "6.14.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
"integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
@ -5550,6 +5560,7 @@
"url": "https://github.com/sponsors/ai"
}
],
"peer": true,
"dependencies": {
"baseline-browser-mapping": "^2.9.0",
"caniuse-lite": "^1.0.30001759",
@ -6347,6 +6358,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -6633,6 +6645,7 @@
"resolved": "https://registry.npmjs.org/cypress/-/cypress-13.15.2.tgz",
"integrity": "sha512-ARbnUorjcCM3XiPwgHKuqsyr5W9Qn+pIIBPaoilnoBkLdSC2oLQjV1BUpnmc7KR+b7Avah3Ly2RMFnfxr96E/A==",
"hasInstallScript": true,
"peer": true,
"dependencies": {
"@cypress/request": "^3.0.6",
"@cypress/xvfb": "^1.2.4",
@ -7393,6 +7406,7 @@
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz",
"integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==",
"peer": true,
"dependencies": {
"ansi-colors": "^4.1.1",
"strip-ansi": "^6.0.1"
@ -7672,6 +7686,7 @@
"version": "8.33.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.33.0.tgz",
"integrity": "sha512-WjOpFQgKK8VrCnAtl8We0SUOy/oVZ5NHykyMiagV1M9r8IFpIJX7DduK6n1mpfhlG7T1NLWm2SuD8QB7KFySaA==",
"peer": true,
"dependencies": {
"@eslint/eslintrc": "^1.4.1",
"@humanwhocodes/config-array": "^0.11.8",
@ -8085,6 +8100,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -10570,6 +10586,7 @@
"version": "27.5.1",
"resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz",
"integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
"peer": true,
"dependencies": {
"@jest/core": "^27.5.1",
"import-local": "^3.0.2",
@ -13214,6 +13231,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -14022,6 +14040,7 @@
"url": "https://tidelift.com/funding/github/npm/postcss"
}
],
"peer": true,
"dependencies": {
"nanoid": "^3.3.4",
"picocolors": "^1.0.0",
@ -15080,6 +15099,7 @@
"version": "6.0.11",
"resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz",
"integrity": "sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g==",
"peer": true,
"dependencies": {
"cssesc": "^3.0.0",
"util-deprecate": "^1.0.2"
@ -15269,6 +15289,7 @@
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
"peer": true,
"dependencies": {
"loose-envify": "^1.4.0",
"object-assign": "^4.1.1",
@ -15459,6 +15480,7 @@
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
"integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
"peer": true,
"dependencies": {
"loose-envify": "^1.1.0"
},
@ -15612,6 +15634,7 @@
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz",
"integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==",
"peer": true,
"dependencies": {
"loose-envify": "^1.1.0",
"scheduler": "^0.23.0"
@ -15723,6 +15746,7 @@
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz",
"integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@ -16228,6 +16252,7 @@
"version": "2.80.0",
"resolved": "https://registry.npmjs.org/rollup/-/rollup-2.80.0.tgz",
"integrity": "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ==",
"peer": true,
"bin": {
"rollup": "dist/bin/rollup"
},
@ -16503,6 +16528,7 @@
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.4.tgz",
"integrity": "sha512-7CxjBoFUfH7fUvtn+SPkkIocthUD9kV3niF1mUMa9TbeyPAf2brtRCZBlT2OpHaXmkscFzGjEfhbJo9gKfotzQ==",
"peer": true,
"dependencies": {
"@babel/runtime": "^7.10.5",
"@fluentui/react-component-event-listener": "~0.63.0",
@ -17581,6 +17607,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -17652,7 +17679,8 @@
"node_modules/three": {
"version": "0.154.0",
"resolved": "https://registry.npmjs.org/three/-/three-0.154.0.tgz",
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug=="
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug==",
"peer": true
},
"node_modules/three-stdlib": {
"version": "2.17.2",
@ -17895,6 +17923,7 @@
"version": "0.21.3",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz",
"integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==",
"peer": true,
"engines": {
"node": ">=10"
},
@ -17948,6 +17977,7 @@
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@ -18259,6 +18289,7 @@
"version": "5.105.0",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz",
"integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==",
"peer": true,
"dependencies": {
"@types/eslint-scope": "^3.7.7",
"@types/estree": "^1.0.8",
@ -18328,6 +18359,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -18377,6 +18409,7 @@
"version": "4.11.1",
"resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz",
"integrity": "sha512-lILVz9tAUy1zGFwieuaQtYiadImb5M3d+H+L1zDYalYoDl0cksAB1UNyuE5MMWJrG6zR1tXkCP2fitl7yoUJiw==",
"peer": true,
"dependencies": {
"@types/bonjour": "^3.5.9",
"@types/connect-history-api-fallback": "^1.3.5",
@ -18431,6 +18464,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -18543,6 +18577,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -18825,6 +18860,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -19264,6 +19300,7 @@
"version": "7.20.12",
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz",
"integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==",
"peer": true,
"requires": {
"@ampproject/remapping": "^2.1.0",
"@babel/code-frame": "^7.18.6",
@ -19813,6 +19850,7 @@
"version": "7.18.6",
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz",
"integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==",
"peer": true,
"requires": {
"@babel/helper-plugin-utils": "^7.18.6"
}
@ -20158,6 +20196,7 @@
"version": "7.20.13",
"resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.20.13.tgz",
"integrity": "sha512-MmTZx/bkUrfJhhYAYt3Urjm+h8DQGrPrnKQ94jLo7NLuOU+T89a7IByhKmrb8SKhrIYIQ0FN0CHMbnFRen4qNw==",
"peer": true,
"requires": {
"@babel/helper-annotate-as-pure": "^7.18.6",
"@babel/helper-module-imports": "^7.18.6",
@ -21423,7 +21462,8 @@
"@popperjs/core": {
"version": "2.11.6",
"resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz",
"integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw=="
"integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==",
"peer": true
},
"@react-three/fiber": {
"version": "8.16.5",
@ -21996,6 +22036,7 @@
"version": "18.3.27",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.27.tgz",
"integrity": "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==",
"peer": true,
"requires": {
"@types/prop-types": "*",
"csstype": "^3.2.2"
@ -22005,6 +22046,7 @@
"version": "18.3.7",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz",
"integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==",
"peer": true,
"requires": {}
},
"@types/react-reconciler": {
@ -22117,6 +22159,7 @@
"version": "5.50.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.50.0.tgz",
"integrity": "sha512-vwksQWSFZiUhgq3Kv7o1Jcj0DUNylwnIlGvKvLLYsq8pAWha6/WCnXUeaSoNNha/K7QSf2+jvmkxggC1u3pIwQ==",
"peer": true,
"requires": {
"@typescript-eslint/scope-manager": "5.50.0",
"@typescript-eslint/type-utils": "5.50.0",
@ -22142,6 +22185,7 @@
"version": "5.50.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.50.0.tgz",
"integrity": "sha512-KCcSyNaogUDftK2G9RXfQyOCt51uB5yqC6pkUYqhYh8Kgt+DwR5M0EwEAxGPy/+DH6hnmKeGsNhiZRQxjH71uQ==",
"peer": true,
"requires": {
"@typescript-eslint/scope-manager": "5.50.0",
"@typescript-eslint/types": "5.50.0",
@ -22391,7 +22435,8 @@
"acorn": {
"version": "8.15.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"peer": true
},
"acorn-globals": {
"version": "6.0.0",
@ -22478,6 +22523,7 @@
"version": "6.14.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz",
"integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
@ -23095,6 +23141,7 @@
"version": "4.28.1",
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz",
"integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==",
"peer": true,
"requires": {
"baseline-browser-mapping": "^2.9.0",
"caniuse-lite": "^1.0.30001759",
@ -23636,6 +23683,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -23844,6 +23892,7 @@
"version": "13.15.2",
"resolved": "https://registry.npmjs.org/cypress/-/cypress-13.15.2.tgz",
"integrity": "sha512-ARbnUorjcCM3XiPwgHKuqsyr5W9Qn+pIIBPaoilnoBkLdSC2oLQjV1BUpnmc7KR+b7Avah3Ly2RMFnfxr96E/A==",
"peer": true,
"requires": {
"@cypress/request": "^3.0.6",
"@cypress/xvfb": "^1.2.4",
@ -24402,6 +24451,7 @@
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz",
"integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==",
"peer": true,
"requires": {
"ansi-colors": "^4.1.1",
"strip-ansi": "^6.0.1"
@ -24620,6 +24670,7 @@
"version": "8.33.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.33.0.tgz",
"integrity": "sha512-WjOpFQgKK8VrCnAtl8We0SUOy/oVZ5NHykyMiagV1M9r8IFpIJX7DduK6n1mpfhlG7T1NLWm2SuD8QB7KFySaA==",
"peer": true,
"requires": {
"@eslint/eslintrc": "^1.4.1",
"@humanwhocodes/config-array": "^0.11.8",
@ -24995,6 +25046,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -26660,6 +26712,7 @@
"version": "27.5.1",
"resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz",
"integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
"peer": true,
"requires": {
"@jest/core": "^27.5.1",
"import-local": "^3.0.2",
@ -28588,6 +28641,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -29152,6 +29206,7 @@
"version": "8.4.21",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz",
"integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==",
"peer": true,
"requires": {
"nanoid": "^3.3.4",
"picocolors": "^1.0.0",
@ -29735,6 +29790,7 @@
"version": "6.0.11",
"resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz",
"integrity": "sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g==",
"peer": true,
"requires": {
"cssesc": "^3.0.0",
"util-deprecate": "^1.0.2"
@ -29874,6 +29930,7 @@
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
"peer": true,
"requires": {
"loose-envify": "^1.4.0",
"object-assign": "^4.1.1",
@ -30016,6 +30073,7 @@
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
"integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
"peer": true,
"requires": {
"loose-envify": "^1.1.0"
}
@ -30129,6 +30187,7 @@
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz",
"integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==",
"peer": true,
"requires": {
"loose-envify": "^1.1.0",
"scheduler": "^0.23.0"
@ -30205,7 +30264,8 @@
"react-refresh": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz",
"integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A=="
"integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==",
"peer": true
},
"react-router": {
"version": "7.12.0",
@ -30566,6 +30626,7 @@
"version": "2.80.0",
"resolved": "https://registry.npmjs.org/rollup/-/rollup-2.80.0.tgz",
"integrity": "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ==",
"peer": true,
"requires": {
"fsevents": "~2.3.2"
}
@ -30751,6 +30812,7 @@
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.4.tgz",
"integrity": "sha512-7CxjBoFUfH7fUvtn+SPkkIocthUD9kV3niF1mUMa9TbeyPAf2brtRCZBlT2OpHaXmkscFzGjEfhbJo9gKfotzQ==",
"peer": true,
"requires": {
"@babel/runtime": "^7.10.5",
"@fluentui/react-component-event-listener": "~0.63.0",
@ -31572,6 +31634,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -31623,7 +31686,8 @@
"three": {
"version": "0.154.0",
"resolved": "https://registry.npmjs.org/three/-/three-0.154.0.tgz",
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug=="
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug==",
"peer": true
},
"three-stdlib": {
"version": "2.17.2",
@ -31820,7 +31884,8 @@
"type-fest": {
"version": "0.21.3",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz",
"integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w=="
"integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==",
"peer": true
},
"type-is": {
"version": "1.6.18",
@ -31857,7 +31922,8 @@
"typescript": {
"version": "4.9.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g=="
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"peer": true
},
"unbox-primitive": {
"version": "1.0.2",
@ -32084,6 +32150,7 @@
"version": "5.105.0",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz",
"integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==",
"peer": true,
"requires": {
"@types/eslint-scope": "^3.7.7",
"@types/estree": "^1.0.8",
@ -32116,6 +32183,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -32179,6 +32247,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -32216,6 +32285,7 @@
"version": "4.11.1",
"resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz",
"integrity": "sha512-lILVz9tAUy1zGFwieuaQtYiadImb5M3d+H+L1zDYalYoDl0cksAB1UNyuE5MMWJrG6zR1tXkCP2fitl7yoUJiw==",
"peer": true,
"requires": {
"@types/bonjour": "^3.5.9",
"@types/connect-history-api-fallback": "^1.3.5",
@ -32252,6 +32322,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@ -32498,6 +32569,7 @@
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
"integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
"peer": true,
"requires": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",

View file

@ -22,6 +22,7 @@ import {SemanticToastContainer} from "react-semantic-toasts-2";
import {FilePreviewProvider} from "./components/FilePreview";
import {TagsProvider} from "./Tags";
import {ZimRoute} from "./components/Zim";
import {DocsRoute} from "./components/Docs";
import ErrorBoundary from "./components/ErrorBoundary";
import {KeyboardShortcutsProvider} from "./components/KeyboardShortcutsProvider";
@ -90,6 +91,7 @@ const router = createBrowserRouter(createRoutesFromElements(<Route
<Route path="more/*" element={<ErrorBoundary><MoreRoute/></ErrorBoundary>}/>
<Route path="inventory/*" element={<ErrorBoundary><InventoryRoute/></ErrorBoundary>}/>
<Route path='archive/*' element={<ErrorBoundary><ArchiveRoute/></ErrorBoundary>}/>
<Route path='docs/*' element={<ErrorBoundary><DocsRoute/></ErrorBoundary>}/>
<Route path='map/*' element={<ErrorBoundary><MapRoute/></ErrorBoundary>}/>
<Route path='zim/*' element={<ErrorBoundary><ZimRoute/></ErrorBoundary>}/>
<Route path='files/*' element={<ErrorBoundary><FilesRoute/></ErrorBoundary>}/>

View file

@ -811,6 +811,38 @@ export async function fetchDomains() {
}
}
export async function fetchAuthors() {
const response = await apiGet(`${COLLECTIONS_API}?kind=author`);
if (response.ok) {
let data = await response.json();
return [data['collections'], data['totals']['collections']];
} else {
const message = await getErrorMessage(response, 'Unable to fetch Authors. See server logs.');
toast({
type: 'error',
title: 'Authors Error',
description: message,
time: 5000,
});
}
}
export async function fetchSubjects() {
const response = await apiGet(`${COLLECTIONS_API}?kind=subject`);
if (response.ok) {
let data = await response.json();
return [data['collections'], data['totals']['collections']];
} else {
const message = await getErrorMessage(response, 'Unable to fetch Subjects. See server logs.');
toast({
type: 'error',
title: 'Subjects Error',
description: message,
time: 5000,
});
}
}
export async function fetchChannels() {
const response = await apiGet(`${COLLECTIONS_API}?kind=channel`);
if (response.ok) {
@ -1378,6 +1410,11 @@ export function downloadArchiveMember(path, member) {
window.open(`${API_URI}/files/zip/download?${params}`, '_blank');
}
export function getArchiveMemberUrl(path, member) {
const params = new URLSearchParams({path, member});
return `${API_URI}/files/zip/download?${params}`;
}
export async function fetchFilesProgress() {
const response = await apiGet(`${API_URI}/files/worker_status`);
if (response.ok) {
@ -1817,6 +1854,8 @@ export async function searchSuggestions(search_str) {
return {
channels: content.channels,
domains: content.domains,
authors: content.authors || [],
subjects: content.subjects || [],
}
} else {
console.error('Failed to get file search suggestions!');
@ -2056,3 +2095,60 @@ export async function fetchSuggestedUserAgent() {
}
return '';
}
// Docs API
const DOCS_API = '/api/docs';
export async function getDocStatistics() {
return await apiGet(`${DOCS_API}/statistics`);
}
export async function getDoc(docId) {
return await apiGet(`${DOCS_API}/${docId}`);
}
export async function getDocByFileGroup(fileGroupId) {
return await apiGet(`${DOCS_API}/view/${fileGroupId}`);
}
export async function deleteDocs(docIds) {
const ids = Array.isArray(docIds) ? docIds.join(',') : docIds;
return await apiDelete(`${DOCS_API}/${ids}`);
}
export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject) {
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
let body = {offset, limit};
if (searchStr) {
body['search_str'] = searchStr;
}
if (order) {
body['order_by'] = order;
}
if (tagNames && tagNames.length > 0) {
body['tag_names'] = tagNames;
}
if (author) {
body['author'] = author;
}
if (subject) {
body['subject'] = subject;
}
console.debug('searching docs', body);
const response = await apiPost(`${DOCS_API}/search`, body);
if (response.ok) {
let data = await response.json();
return [data['file_groups'], data['totals']['file_groups']];
} else {
const message = await getErrorMessage(response, 'Cannot search docs. See server logs.');
toast({
type: 'error',
title: 'Unable to search docs',
description: message,
time: 5000,
});
throw Error(message);
}
}

View file

@ -346,6 +346,8 @@ function ArchivePage() {
export function ArchiveCard({file}) {
const {s} = useContext(ThemeContext);
const {sort} = useSearchOrder();
const sortField = sort ? sort.replace(/^-/, '') : null;
const {data, directory} = file;
// Resolve data paths (filename-only) to full relative paths using directory
@ -386,7 +388,16 @@ export function ArchiveCard({file}) {
<span {...s}>{domain}</span>
</CardLink>}
<CardMeta {...s}>
{isoDatetimeToAgoPopup(dt, false)}
{sortField === 'published_modified_datetime'
? isoDatetimeToAgoPopup(file.published_modified_datetime, false)
: sortField === 'download_datetime'
? isoDatetimeToAgoPopup(file.download_datetime, false)
: sortField === 'size'
? humanFileSize(file.size)
: sortField === 'viewed'
? isoDatetimeToAgoPopup(file.viewed, false)
: isoDatetimeToAgoPopup(dt, false)
}
</CardMeta>
<CardDescription>
<Link to={`/archive/${data.id}`}>

View file

@ -0,0 +1,152 @@
import React, {useCallback, useEffect, useRef, useState} from 'react';
import {getArchiveContents, getArchiveMemberUrl} from '../api';
import {Icon, Label, Loader, Message} from 'semantic-ui-react';
import {Button} from './Theme';
import {useLocalStorage} from './Common';
const IMAGE_EXTENSIONS = /\.(jpe?g|png|gif|webp|bmp|tiff?)$/i;
/**
* Recursively flatten archive tree entries into a list of file paths.
*/
function flattenEntries(entries, prefix = '') {
let paths = [];
for (const entry of entries) {
const fullPath = entry.path || (prefix ? `${prefix}/${entry.name}` : entry.name);
if (entry.is_dir && entry.children) {
paths = paths.concat(flattenEntries(entry.children, fullPath));
} else if (!entry.is_dir) {
paths.push(fullPath);
}
}
return paths;
}
export function CbzViewer({path}) {
const [pages, setPages] = useState(null);
const [currentPage, setCurrentPage] = useState(0);
const [rtl, setRtl] = useLocalStorage('cbzViewerRtl', false);
const [fullscreen, setFullscreen] = useState(false);
const [error, setError] = useState(null);
useEffect(() => {
const load = async () => {
try {
const contents = await getArchiveContents(path);
if (!contents) {
setError('Could not read archive contents.');
return;
}
const images = flattenEntries(contents.entries)
.filter(name => IMAGE_EXTENSIONS.test(name))
.sort();
const urls = images.map(member => getArchiveMemberUrl(path, member));
setPages(urls);
setCurrentPage(0);
} catch (e) {
setError(e.message || 'Could not read archive contents.');
}
};
load();
}, [path]);
const goNext = useCallback(() => {
setCurrentPage(p => Math.min(p + 1, (pages?.length || 1) - 1));
}, [pages]);
const goPrev = useCallback(() => {
setCurrentPage(p => Math.max(p - 1, 0));
}, []);
const onLeftClick = rtl ? goNext : goPrev;
const onRightClick = rtl ? goPrev : goNext;
const imageContainerRef = useRef(null);
const handleFullscreen = useCallback(() => {
setFullscreen(f => !f);
}, []);
useEffect(() => {
const handleKey = (e) => {
if (e.key === 'ArrowLeft') onLeftClick();
else if (e.key === 'ArrowRight') onRightClick();
else if (e.key === 'Escape') setFullscreen(false);
};
window.addEventListener('keydown', handleKey);
return () => window.removeEventListener('keydown', handleKey);
}, [onLeftClick, onRightClick]);
if (error) {
return <Message negative>
<Message.Header>Cannot load comic</Message.Header>
<p>{error}</p>
</Message>;
}
if (!pages) return <Loader active inline='centered'/>;
if (pages.length === 0) {
return <Message warning>
<Message.Header>No images found</Message.Header>
<p>This archive does not contain any image files.</p>
</Message>;
}
const hasPrev = currentPage > 0;
const hasNext = currentPage < pages.length - 1;
return <div style={{marginBottom: '1em'}}>
<div ref={imageContainerRef} style={{
display: 'flex',
justifyContent: 'center',
alignItems: 'center',
background: '#222',
borderRadius: fullscreen ? 0 : '4px',
padding: '0.5em',
minHeight: '300px',
position: fullscreen ? 'fixed' : 'relative',
...(fullscreen ? {top: 0, left: 0, right: 0, bottom: 0, zIndex: 9999} : {}),
}}>
<img
src={pages[currentPage]}
alt={`Page ${currentPage + 1}`}
style={{
maxHeight: fullscreen ? '100vh' : '75vh',
maxWidth: '100%',
objectFit: 'contain',
}}
/>
<div style={{
position: 'absolute',
top: 0, left: 0, right: 0, bottom: 0,
display: 'grid',
gridTemplateColumns: '1fr 1fr 1fr',
}}>
<div style={{cursor: 'pointer'}} onClick={onLeftClick}/>
<div style={{cursor: 'pointer'}} onClick={handleFullscreen}/>
<div style={{cursor: 'pointer'}} onClick={onRightClick}/>
</div>
</div>
<div style={{
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
gap: '0.5em',
marginTop: '0.5em',
}}>
<Button icon basic active={rtl} onClick={() => setRtl(r => !r)}
aria-label='Reading direction'>
<Icon name={rtl ? 'hand point left' : 'hand point right'}/>
</Button>
<span style={{flex: 1}}/>
<Button icon basic disabled={rtl ? !hasNext : !hasPrev}
onClick={onLeftClick} aria-label='Previous page'>
<Icon name='chevron left'/>
</Button>
<Label>{currentPage + 1} / {pages.length}</Label>
<Button icon basic disabled={rtl ? !hasPrev : !hasNext}
onClick={onRightClick} aria-label='Next page'>
<Icon name='chevron right'/>
</Button>
</div>
</div>;
}

View file

@ -19,7 +19,7 @@ import {
Search,
Transition
} from 'semantic-ui-react';
import {Link, NavLink, useNavigate, useSearchParams} from "react-router";
import {Link, NavLink, useNavigate} from "react-router";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useHotspot, useSearchDirectories, useSearchOrder, useThrottle, useWROLMode} from "../hooks/customHooks";
import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts";
@ -701,15 +701,9 @@ export function textEllipsis(str, maxLength = 100, {side = "end", ellipsis = "..
}
export function TabLinks({links}) {
const [searchParams] = useSearchParams();
const getTo = (to) => {
return `${to}?${searchParams.toString()}`;
}
return <Menu tabular>
{links.map((link) => <NavLink
to={getTo(link.to)}
to={link.to}
style={{padding: '1em'}}
key={link.to}
end={link.end === true ? true : null}
@ -1084,11 +1078,12 @@ export function emptyToNull(obj) {
return obj;
}
export function mimetypeColor(mimetype) {
export function mimetypeColor(mimetype, path = '') {
if (!mimetype) {
return 'grey';
}
try {
const lowerPath = path.toLowerCase();
if (mimetype === 'application/pdf') {
return 'red'
} else if (mimetype.startsWith('video/')) {
@ -1099,6 +1094,11 @@ export function mimetypeColor(mimetype) {
return 'green'
} else if (mimetype.startsWith('application/epub')) {
return 'yellow'
} else if (mimetype.includes('cbz') || mimetype.includes('cbr')
|| mimetype.includes('comicbook')
|| lowerPath.endsWith('.cbz') || lowerPath.endsWith('.cbr')
|| lowerPath.endsWith('.cbt') || lowerPath.endsWith('.cb7')) {
return 'yellow'
} else if (mimetype.startsWith('audio/')) {
return 'violet'
} else if (isZipMimetype(mimetype)) {
@ -1142,7 +1142,8 @@ export function mimetypeIconName(mimetype, lowerPath = '') {
return 'file archive';
} else if (mimetype.startsWith('application/x-iso9660-image') || mimetype.startsWith('application/x-raw-disk-image') || mimetype.startsWith('application/x-cd-image')) {
return 'dot circle';
} else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook') || mimetype.startsWith('application/vnd.amazon.mobi8-ebook')) {
} else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook') || mimetype.startsWith('application/vnd.amazon.mobi8-ebook')
|| mimetype.startsWith('application/vnd.comicbook') || mimetype.startsWith('application/x-cbz') || mimetype.startsWith('application/x-cbr')) {
return 'book';
} else if (mimetype.startsWith('text/vtt') || mimetype.startsWith('text/srt')) {
return 'closed captioning';
@ -1527,6 +1528,10 @@ export const filterToMimetypes = (filter) => {
return ['audio'];
} else if (filter === 'ebook') {
return ['application/epub+zip', 'application/x-mobipocket-ebook'];
} else if (filter === 'doc') {
return ['application/epub+zip', 'application/x-mobipocket-ebook', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.oasis.opendocument.text', 'application/x-cbz', 'application/x-cbr'];
} else if (filter === 'image') {
return ['image'];
} else if (filter === 'zip') {

532
app/src/components/Docs.js Normal file
View file

@ -0,0 +1,532 @@
import React, {useContext, useEffect, useState} from 'react';
import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router";
import {Grid, Icon as SIcon, Label, Loader, StatisticLabel, StatisticValue} from "semantic-ui-react";
import {deleteDocs, getDocStatistics, tagFileGroup, untagFileGroup} from "../api";
import {Media, ThemeContext} from "../contexts/contexts";
import {
APIButton,
BackButton,
encodeMediaPath,
ErrorMessage,
humanFileSize,
isoDatetimeToAgoPopup,
PageContainer,
PreviewPath,
SearchInput,
SortButton,
TabLinks,
toLocaleString,
useTitle
} from "./Common";
import {Button, darkTheme, Header, Icon, Segment, Statistic, Tab} from "./Theme";
import {FilesView} from "./Files";
import {useAuthors, useDoc, useOneQuery, useSearchDocs, useSubjects} from "../hooks/customHooks";
import {TagsSelector} from "../Tags";
import {toast} from "react-semantic-toasts-2";
import {CollectionTable} from "./collections/CollectionTable";
import {CbzViewer} from "./CbzViewer";
import _ from "lodash";
function DocsPage() {
useTitle('Documents');
const searchInputRef = React.useRef();
const {
docs,
totalPages,
activePage,
setPage,
searchStr,
setSearchStr,
author,
subject,
fetchDocs,
} = useSearchDocs();
const [selectedDocs, setSelectedDocs] = useState([]);
const onSelect = (path, checked) => {
if (checked && path) {
setSelectedDocs([...selectedDocs, path]);
} else if (path) {
setSelectedDocs(selectedDocs.filter(i => i !== path));
}
}
const onDelete = async () => {
const docIds = docs.filter(i => selectedDocs.indexOf(i['primary_path']) >= 0).map(i => i['id']);
await deleteDocs(docIds);
await fetchDocs();
setSelectedDocs([]);
}
const invertSelection = () => {
const newSelectedDocs = docs.map(doc => doc['key']).filter(i => selectedDocs.indexOf(i) < 0);
setSelectedDocs(newSelectedDocs);
}
const clearSelection = (e) => {
if (e) e.preventDefault();
setSelectedDocs([]);
}
const selectElm = <div style={{marginTop: '0.5em'}}>
<APIButton
color='red'
disabled={_.isEmpty(selectedDocs)}
confirmButton='Delete'
confirmContent='Are you sure you want to delete these documents? This cannot be undone.'
onClick={onDelete}
obeyWROLMode={true}
>Delete</APIButton>
<Button color='grey' onClick={invertSelection} disabled={_.isEmpty(docs)}>Invert</Button>
<Button color='yellow' onClick={clearSelection}
disabled={_.isEmpty(docs) || _.isEmpty(selectedDocs)}>Clear</Button>
</div>;
let docOrders = [
{value: 'published_datetime', text: 'Published Date', short: 'P.Date'},
{value: 'size', text: 'Size'},
{value: 'title', text: 'Title'},
];
if (searchStr) {
docOrders = [{value: 'rank', text: 'Rank'}, ...docOrders];
}
const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView(
{
files: docs,
activePage: activePage,
totalPages: totalPages,
selectElem: selectElm,
selectedKeys: selectedDocs,
onSelect: onSelect,
setPage: setPage,
},
);
const [localSearchStr, setLocalSearchStr] = React.useState(searchStr || '');
const searchInput = <SearchInput
onChange={setLocalSearchStr}
onClear={() => setLocalSearchStr(null)}
searchStr={localSearchStr}
onSubmit={setSearchStr}
placeholder='Search Documents...'
inputRef={searchInputRef}
/>;
return <>
{author && <Header as='h1'>Author: {author}</Header>}
{subject && <Header as='h1'>Subject: {subject}</Header>}
<Media at='mobile'>
<Grid>
<Grid.Row>
<Grid.Column width={2}>{selectButton}</Grid.Column>
<Grid.Column width={2}>{viewButton}</Grid.Column>
<Grid.Column width={4}>{limitDropdown}</Grid.Column>
<Grid.Column width={2}>{tagQuerySelector}</Grid.Column>
<Grid.Column width={6}><SortButton sorts={docOrders}/></Grid.Column>
</Grid.Row>
<Grid.Row width={16}>
<Grid.Column>{searchInput}</Grid.Column>
</Grid.Row>
</Grid>
</Media>
<Media greaterThanOrEqual='tablet'>
<Grid>
<Grid.Row>
<Grid.Column width={1}>{selectButton}</Grid.Column>
<Grid.Column width={1}>{viewButton}</Grid.Column>
<Grid.Column width={2}>{limitDropdown}</Grid.Column>
<Grid.Column width={1}>{tagQuerySelector}</Grid.Column>
<Grid.Column width={3}><SortButton sorts={docOrders}/></Grid.Column>
<Grid.Column width={8}>{searchInput}</Grid.Column>
</Grid.Row>
</Grid>
</Media>
{body}
{paginator}
</>;
}
function DocPage() {
const {fileGroupId} = useParams();
const navigate = useNavigate();
const {docFile, doc, fetchDoc} = useDoc(parseInt(fileGroupId));
const {theme} = useContext(ThemeContext);
const title = docFile ? (docFile.title || docFile.name) : null;
useTitle(title);
if (docFile === null) {
return <Segment><Loader active/></Segment>;
}
if (docFile === undefined) {
return <>
<Header as='h2'>Unknown document</Header>
<Header as='h4'>This document does not exist</Header>
</>;
}
const {data, size, directory} = docFile;
// Cover image.
const coverPath = data && data.cover_path ? data.cover_path : null;
const coverUrl = coverPath ? `/media/${encodeMediaPath(coverPath)}` : null;
// Primary file URLs.
const primaryPath = docFile.primary_path;
const downloadUrl = primaryPath ? `/download/${encodeMediaPath(primaryPath)}` : null;
const isEpub = docFile.mimetype && docFile.mimetype.startsWith('application/epub');
const mt = docFile.mimetype || '';
const primaryExt = (primaryPath || '').toLowerCase().split('.').pop();
const isCbz = mt.includes('cbz') || mt.includes('cbr') || mt.includes('comicbook+zip') || mt.includes('comicbook-rar')
|| ['cbz', 'cbr', 'cbt', 'cb7'].includes(primaryExt);
const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
const openUrl = viewerUrl || (primaryPath ? `/media/${encodeMediaPath(primaryPath)}` : null);
const handleDelete = async () => {
if (doc && doc.id) {
await deleteDocs([doc.id]);
toast({
type: 'success',
title: 'Document Deleted',
description: 'Document was successfully deleted.',
time: 2000,
});
navigate(-1);
}
};
const localAddTag = async (name) => {
await tagFileGroup(docFile, name);
await fetchDoc();
};
const localRemoveTag = async (name) => {
await untagFileGroup(docFile, name);
await fetchDoc();
};
const publishedDatetimeString = docFile.published_datetime
? isoDatetimeToAgoPopup(docFile.published_datetime, true)
: null;
// Format badge based on mimetype.
const formatLabel = () => {
const mt = docFile.mimetype || '';
if (mt.startsWith('application/epub')) return 'EPUB';
if (mt === 'application/x-mobipocket-ebook') return 'MOBI';
if (mt === 'application/pdf') return 'PDF';
if (mt.includes('wordprocessingml')) return 'DOCX';
if (mt === 'application/msword') return 'DOC';
if (mt.includes('opendocument.text')) return 'ODT';
if (mt.includes('cbz') || mt.includes('comicbook+zip')) return 'CBZ';
if (mt.includes('cbr') || mt.includes('comicbook-rar')) return 'CBR';
if (mt.includes('cbt')) return 'CBT';
if (mt.includes('cb7') || mt.includes('x-7z')) return 'CB7';
return mt.split('/').pop().toUpperCase();
};
// About pane.
const aboutPane = {
menuItem: 'About', render: () => <Tab.Pane>
{doc && doc.description && <>
<Header as='h3'>Description</Header>
<p>{doc.description}</p>
</>}
{doc && doc.publisher && <>
<Header as='h3'>Publisher</Header>
<p>{doc.publisher}</p>
</>}
{doc && doc.language && <>
<Header as='h3'>Language</Header>
<p>{doc.language}</p>
</>}
{doc && doc.subject && <>
<Header as='h3'>Subject</Header>
<p><Link to={`/docs?subject=${encodeURIComponent(doc.subject)}`}>{doc.subject}</Link></p>
</>}
{doc && doc.page_count && <>
<Header as='h3'>Page Count</Header>
<p>{doc.page_count}</p>
</>}
<Header as='h3'>Size</Header>
<p>{humanFileSize(size)}</p>
</Tab.Pane>
};
// Files pane.
const filesPane = {
menuItem: 'Files', render: () => <Tab.Pane>
{docFile.files && docFile.files.map((file, idx) =>
<div key={idx} style={{marginBottom: '0.5em'}}>
<PreviewPath path={file.path} mimetype={file.mimetype} taggable={false}>
{file.path}
</PreviewPath>
{file.size && <span style={{marginLeft: '1em', color: 'grey'}}>
({humanFileSize(file.size)})
</span>}
</div>
)}
<Header as='h3'>Directory</Header>
<p>{directory || 'Unknown'}</p>
</Tab.Pane>
};
const tabPanes = [aboutPane, filesPane];
const tabMenu = theme === darkTheme ? {inverted: true, attached: true} : {attached: true};
// Inline reader: epub viewer iframe or PDF iframe.
const isPdf = docFile.mimetype === 'application/pdf';
const canEmbed = isEpub || isPdf;
const embedUrl = isEpub ? viewerUrl : (isPdf && primaryPath ? `/media/${encodeMediaPath(primaryPath)}` : null);
return <>
<BackButton/>
{isCbz && primaryPath && <CbzViewer path={primaryPath}/>}
{canEmbed && embedUrl && !isCbz && <div style={{marginBottom: '1em'}}>
<iframe
src={embedUrl}
title={docFile.title || docFile.name}
style={{width: '100%', height: '80vh', border: '1px solid #ccc', borderRadius: '4px'}}
/>
</div>}
<Segment>
<Header as='h2'>{docFile.title || docFile.name}</Header>
{docFile.author && <Header as='h3'>
Author: <Link to={`/docs?author=${encodeURIComponent(docFile.author)}`}>{docFile.author}</Link>
</Header>}
<Grid columns={2} stackable>
<Grid.Row>
{publishedDatetimeString && <Grid.Column>
<Header as='h4'>Published: {publishedDatetimeString}</Header>
</Grid.Column>}
<Grid.Column>
<Label>{formatLabel()}</Label>
{doc && doc.page_count && <Label>{doc.page_count} pages</Label>}
</Grid.Column>
</Grid.Row>
</Grid>
<div style={{marginTop: '1em'}}>
{openUrl && <Button as='a' href={openUrl} target='_blank' rel='noreferrer' color='violet'>
<SIcon name='expand arrows alternate'/> Open
</Button>}
{downloadUrl && <Button as='a' href={downloadUrl}>
<SIcon name='download'/> Download
</Button>}
<APIButton
color='red'
confirmContent='Are you sure you want to delete this document? All files will be deleted.'
confirmButton='Delete'
onClick={handleDelete}
obeyWROLMode={true}
>Delete</APIButton>
</div>
</Segment>
<Segment>
<TagsSelector selectedTagNames={docFile.tags} onAdd={localAddTag} onRemove={localRemoveTag}/>
</Segment>
<Tab menu={tabMenu} panes={tabPanes}/>
</>;
}
const AUTHOR_COLUMNS = [
{key: 'name', label: 'Author', sortable: true, width: 10},
{key: 'item_count', label: 'Documents', sortable: true, align: 'right', width: 6},
];
const AUTHOR_ROUTES = {
search: '/docs',
searchParam: 'author',
};
const SUBJECT_COLUMNS = [
{key: 'name', label: 'Subject', sortable: true, width: 10},
{key: 'item_count', label: 'Documents', sortable: true, align: 'right', width: 6},
];
const SUBJECT_ROUTES = {
search: '/docs',
searchParam: 'subject',
};
function AuthorsPage() {
useTitle('Doc Authors');
const [authors] = useAuthors();
const [searchStr, setSearchStr] = useOneQuery('author');
const searchInputRef = React.useRef();
const header = <div style={{marginBottom: '1em'}}>
<Grid stackable columns={2}>
<Grid.Row>
<Grid.Column>
<SearchInput
placeholder='Author filter...'
size='large'
searchStr={searchStr}
disabled={!Array.isArray(authors) || authors.length === 0}
onClear={() => setSearchStr('')}
onChange={setSearchStr}
onSubmit={null}
inputRef={searchInputRef}
/>
</Grid.Column>
</Grid.Row>
</Grid>
</div>;
return <>
{header}
<CollectionTable
collections={authors}
columns={AUTHOR_COLUMNS}
routes={AUTHOR_ROUTES}
searchStr={searchStr}
emptyMessage='No authors found. Index some documents with author metadata!'
/>
</>;
}
function SubjectsPage() {
useTitle('Doc Subjects');
const [subjects] = useSubjects();
const [searchStr, setSearchStr] = useOneQuery('subject');
const searchInputRef = React.useRef();
const header = <div style={{marginBottom: '1em'}}>
<Grid stackable columns={2}>
<Grid.Row>
<Grid.Column>
<SearchInput
placeholder='Subject filter...'
size='large'
searchStr={searchStr}
disabled={!Array.isArray(subjects) || subjects.length === 0}
onClear={() => setSearchStr('')}
onChange={setSearchStr}
onSubmit={null}
inputRef={searchInputRef}
/>
</Grid.Column>
</Grid.Row>
</Grid>
</div>;
return <>
{header}
<CollectionTable
collections={subjects}
columns={SUBJECT_COLUMNS}
routes={SUBJECT_ROUTES}
searchStr={searchStr}
emptyMessage='No subjects found. Index some documents with subject metadata!'
/>
</>;
}
function DocsStatisticsPage() {
useTitle('Doc Statistics');
const [statistics, setStatistics] = useState(null);
useEffect(() => {
const fetchStats = async () => {
try {
const response = await getDocStatistics();
if (response.ok) {
const data = await response.json();
setStatistics(data.statistics);
}
} catch (e) {
setStatistics(undefined);
}
};
fetchStats();
}, []);
if (statistics === null) {
return <Loader active inline='centered'/>;
}
if (statistics === undefined) {
return <ErrorMessage>Unable to fetch Doc Statistics</ErrorMessage>;
}
return <Segment>
<Header as='h1' textAlign='center'>Documents</Header>
<Statistic.Group>
<Statistic style={{margin: '2em'}}>
<StatisticValue>{toLocaleString(statistics.doc_count)}</StatisticValue>
<StatisticLabel>Documents</StatisticLabel>
</Statistic>
<Statistic style={{margin: '2em'}}>
<StatisticValue>{toLocaleString(statistics.epub_count)}</StatisticValue>
<StatisticLabel>eBooks</StatisticLabel>
</Statistic>
<Statistic style={{margin: '2em'}}>
<StatisticValue>{toLocaleString(statistics.pdf_count)}</StatisticValue>
<StatisticLabel>PDFs</StatisticLabel>
</Statistic>
<Statistic style={{margin: '2em'}}>
<StatisticValue>{toLocaleString(statistics.other_count)}</StatisticValue>
<StatisticLabel>Other</StatisticLabel>
</Statistic>
<Statistic style={{margin: '2em'}}>
<StatisticValue>{humanFileSize(statistics.total_size)}</StatisticValue>
<StatisticLabel>Total Size</StatisticLabel>
</Statistic>
<Statistic style={{margin: '2em'}}>
<StatisticValue>{toLocaleString(statistics.author_count)}</StatisticValue>
<StatisticLabel>Authors</StatisticLabel>
</Statistic>
<Statistic style={{margin: '2em'}}>
<StatisticValue>{toLocaleString(statistics.subject_count)}</StatisticValue>
<StatisticLabel>Subjects</StatisticLabel>
</Statistic>
</Statistic.Group>
</Segment>;
}
export function DocsRoute() {
const location = useLocation();
const path = location.pathname;
const links = [
{
text: 'Docs',
to: '/docs',
end: true,
isActive: () => path === '/docs' || /^\/docs\/view\/\d+$/.test(path)
},
{text: 'Subjects', to: '/docs/subjects', isActive: () => path.startsWith('/docs/subjects')},
{text: 'Authors', to: '/docs/authors', isActive: () => path.startsWith('/docs/authors')},
{text: 'Statistics', to: '/docs/statistics'},
];
return <PageContainer>
<TabLinks links={links}/>
<Routes>
<Route path='/' element={<DocsPage/>}/>
<Route path='view/:fileGroupId' element={<DocPage/>}/>
<Route path='subjects' element={<SubjectsPage/>}/>
<Route path='authors' element={<AuthorsPage/>}/>
<Route path='statistics' element={<DocsStatisticsPage/>}/>
</Routes>
</PageContainer>;
}

View file

@ -6,6 +6,7 @@ import {Media} from "../contexts/contexts";
import {encodeMediaPath, isSupportedArchive} from "./Common";
import {getFile, tagFileGroup, untagFileGroup} from "../api";
import {ArchivePreviewContent} from "./ArchivePreview";
import {CbzViewer} from "./CbzViewer";
import {StlViewer} from "react-stl-viewer";
import {Button, Modal} from "./Theme";
import {toast} from "react-semantic-toasts-2";
@ -361,6 +362,9 @@ export function FilePreviewProvider({children}) {
setModalContent(getAudioPreviewModal(previewFile), url, downloadURL, path, taggable);
} else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.stl')) {
setModalContent(getSTLPreviewModal(previewFile), null, downloadURL, path, taggable);
} else if (mimetype.includes('cbz') || mimetype.includes('cbr') || mimetype.includes('comicbook+zip') || mimetype.includes('comicbook-rar')
|| lowerPath.endsWith('.cbz') || lowerPath.endsWith('.cbr') || lowerPath.endsWith('.cbt') || lowerPath.endsWith('.cb7')) {
setModalContent(<Modal.Content><CbzViewer path={path}/></Modal.Content>, null, downloadURL, path, taggable);
} else if (isSupportedArchive(mimetype, lowerPath)) {
setModalContent(<ArchivePreviewContent previewFile={previewFile}/>, null, downloadURL, path, taggable);
} else {

View file

@ -37,12 +37,13 @@ import {
usePages,
useSearchFiles,
useSearchFilter,
useSearchOrder,
useSearchView,
useStatusFlag,
useWROLMode
} from "../hooks/customHooks";
import {useFileWorkerStatus} from "../contexts/FileWorkerStatusContext";
import {Route, Routes} from "react-router";
import {Link, Route, Routes} from "react-router";
import {CardPlaceholder} from "./Placeholder";
import {ArchiveCard, ArchiveRowCells} from "./Archive";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
@ -66,20 +67,23 @@ import {Headlines} from "./Headline";
import {useSearch} from "./Search";
import {FILES_MEDIA_URI} from "./Vars";
function EbookCard({file}) {
const {s} = useContext(ThemeContext);
function EbookCard({file, sortField}) {
const {s, t} = useContext(ThemeContext);
const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
const isEpub = file['mimetype'].startsWith('application/epub');
const isEpub = file['mimetype'] && file['mimetype'].startsWith('application/epub');
const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
const color = mimetypeColor(file.mimetype);
// Link to doc detail page if this is a modeled doc.
const detailUrl = file.model === 'doc' ? `/docs/view/${file.id}` : null;
const color = mimetypeColor(file.mimetype, file.primary_path);
const title = file.title || file.stem || file.name;
const header = <ExternalCardLink to={viewerUrl || downloadUrl} className='card-title-ellipsis'>
{title}
</ExternalCardLink>;
const header = detailUrl
? <Link to={detailUrl} className='no-link-underscore card-link card-title-ellipsis' {...t}>{title}</Link>
: <ExternalCardLink to={viewerUrl || downloadUrl} className='card-title-ellipsis'>{title}</ExternalCardLink>;
return <Card color={color}>
<CardPoster file={file} preview={true}/>
<CardPoster file={file} to={detailUrl}/>
<CardContent {...s}>
<CardHeader>
<Container textAlign='left'>
@ -90,7 +94,10 @@ function EbookCard({file}) {
</CardHeader>
<CardMeta>
{file.author ? <b {...s}>{file.author}</b> : null}
{file.size && <p {...s}>{humanFileSize(file.size)}</p>}
{sortField === 'published_datetime'
? <p {...s}>{file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime, false) : null}</p>
: file.size && <p {...s}>{humanFileSize(file.size)}</p>
}
</CardMeta>
</CardContent>
</Card>
@ -128,10 +135,14 @@ function ImageCard({file}) {
function FileCard({file}) {
const {s} = useContext(ThemeContext);
const {sort} = useSearchOrder();
const sortField = sort ? sort.replace(/^-/, '') : null;
const isEbookType = file.mimetype && (
const isDocType = file.model === 'doc' || (file.mimetype && (
file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
);
));
// Default doc sort is published_datetime when no order query param is set.
const docSortField = sortField || 'published_datetime';
if (file.model === 'video' && 'video' in file) {
return <VideoCard key={file['primary_path']} file={file}/>;
@ -139,12 +150,12 @@ function FileCard({file}) {
return <ArchiveCard key={file['primary_path']} file={file}/>;
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
return <ImageCard key={file['primary_path']} file={file}/>;
} else if (isEbookType) {
return <EbookCard key={file['primary_path']} file={file}/>;
} else if (isDocType) {
return <EbookCard key={file['primary_path']} file={file} sortField={docSortField}/>;
}
const author = file.author;
const color = mimetypeColor(file.mimetype);
const color = mimetypeColor(file.mimetype, file.primary_path);
const size = file.size !== null && file.size !== undefined ? humanFileSize(file.size) : null;
const title = file.title || file.name || file.primary_path;
@ -160,8 +171,13 @@ function FileCard({file}) {
</PreviewLink>
</CardHeader>
{author && <b {...s}>{author}</b>}
<p>{isoDatetimeToAgoPopup(dt, false)}</p>
<p>{size}</p>
{sortField === 'published_datetime'
? <p>{file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime, false) : null}</p>
: <>
<p>{isoDatetimeToAgoPopup(dt, false)}</p>
<p>{size}</p>
</>
}
</CardContent>
</Card>
}
@ -206,7 +222,7 @@ function ImageRowCells({file}) {
}
export function EbookRowCells({file}) {
let cover = <Card.Icon><FileIcon file={file}/></Card.Icon>;
let cover = <FileIcon file={file} size='large'/>;
const posterPath = findPosterPath(file);
if (posterPath) {
const coverSrc = `/media/${encodeMediaPath(posterPath)}`;
@ -229,9 +245,9 @@ export function EbookRowCells({file}) {
}
function FileRow({file}) {
const isEbookType = file.mimetype && (
const isDocType = file.model === 'doc' || (file.mimetype && (
file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
);
));
if (file.model === 'video' && 'video' in file) {
return <VideoRowCells file={file}/>;
@ -239,7 +255,7 @@ function FileRow({file}) {
return <ArchiveRowCells file={file}/>;
} else if (file.mimetype && file.mimetype.startsWith('image/')) {
return <ImageRowCells file={file}/>;
} else if (isEbookType) {
} else if (isDocType) {
return <EbookRowCells key={file['primary_path']} file={file}/>;
}
@ -480,6 +496,7 @@ export function FileSearchFilterButton({size = 'medium'}) {
{key: 'archive', text: 'Archive', value: 'archive'},
{key: 'pdf', text: 'PDF', value: 'pdf'},
{key: 'ebook', text: 'eBook', value: 'ebook'},
{key: 'doc', text: 'Document', value: 'doc'},
{key: 'audio', text: 'Audio', value: 'audio'},
{key: 'image', text: 'Image', value: 'image'},
{key: 'zip', text: 'ZIP', value: 'zip'},

View file

@ -37,6 +37,7 @@ function updateFavicon(colorName) {
const links = [
{text: 'Videos', to: '/videos', key: 'videos'},
{text: 'Archive', to: '/archive', key: 'archive'},
{text: 'Docs', to: '/docs', key: 'docs'},
{text: 'Map', to: '/map', key: 'map'},
{text: 'Zim', to: '/zim', key: 'zim'},
{text: 'Files', to: '/files', key: 'files'},

View file

@ -64,7 +64,7 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
const pages = usePages(defaultLimit, totalPages);
// text/html, video*, image*, etc.
const filter = searchParams.get('filter');
// archive/video/ebook/etc.
// archive/video/doc/etc.
const model_ = searchParams.get('model') || model;
const anySearch = (!datesIsEmpty) || searchStr || (activeTags && activeTags.length > 0) || filter || model;
@ -121,6 +121,8 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
fileGroups: [],
channels: [],
domains: [],
authors: [],
subjects: [],
zimsEstimates: [],
}
const [suggestions, setSuggestions] = React.useState(defaultSuggestions);
@ -162,6 +164,8 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
...prevState,
channels: generalData.channels,
domains: generalData.domains,
authors: generalData.authors,
subjects: generalData.subjects,
}
});
}
@ -274,6 +278,30 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
})
}
}
if (newSuggestions.authors && newSuggestions.authors.length > 0) {
results.authors = {
name: 'Authors', results: newSuggestions.authors.map(i => {
return {
type: 'author',
title: i.name,
id: i.id,
location: `/docs?author=${encodeURIComponent(i.name)}`
}
})
}
}
if (newSuggestions.subjects && newSuggestions.subjects.length > 0) {
results.subjects = {
name: 'Subjects', results: newSuggestions.subjects.map(i => {
return {
type: 'subject',
title: i.name,
id: i.id,
location: `/docs?subject=${encodeURIComponent(i.name)}`
}
})
}
}
// Match at most 5 Tags.
const matchingTags = searchStr ? fuzzyMatchTagsByName(searchStr).slice(0, 5) : null;
@ -304,6 +332,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
otherSum: otherSum,
channels: newSuggestions.channels.length,
domains: newSuggestions.domains.length,
authors: newSuggestions.authors?.length,
subjects: newSuggestions.subjects?.length,
tags: matchingTags?.length,
apps: matchingApps?.length,
});

View file

@ -854,6 +854,8 @@ export function VideosRoute(props) {
export function VideoCard({file}) {
const {video} = file;
const {s} = useContext(ThemeContext);
const {sort} = useSearchOrder();
const sortField = sort ? sort.replace(/^-/, '') : null;
// Default to video FilePreview for lone video files.
let video_url;
@ -904,7 +906,18 @@ export function VideoCard({file}) {
{channel && <Link to={channel_url} className="no-link-underscore card-link">
<b {...s}>{channel.name}</b>
</Link>}
<p {...s}>{isoDatetimeToAgoPopup(file.published_datetime, false)}</p>
{sortField === 'length'
? <p {...s}>{secondsToFullDuration(file.length || 0)}</p>
: sortField === 'size'
? <p {...s}>{humanFileSize(file.size)}</p>
: sortField === 'view_count'
? <p {...s}>{humanNumber(video.view_count || 0)} views</p>
: sortField === 'viewed'
? <p {...s}>{isoDatetimeToAgoPopup(file.viewed, false)}</p>
: sortField === 'download_datetime'
? <p {...s}>{isoDatetimeToAgoPopup(file.download_datetime, false)}</p>
: <p {...s}>{isoDatetimeToAgoPopup(file.published_datetime, false)}</p>
}
</Container>
</CardDescription>
</CardContent>

View file

@ -4,9 +4,12 @@ import {
createChannel,
fetchChannels,
fetchDecoded,
fetchAuthors,
fetchDomains,
fetchSubjects,
filesSearch,
getArchive,
getDocByFileGroup,
getArchiveStatistics,
getChannel,
getConfigs,
@ -28,6 +31,7 @@ import {
searchArchives,
searchChannels,
searchDirectories,
searchDocs,
searchVideos,
searchZim,
updateChannel,
@ -294,6 +298,81 @@ export const useDomains = () => {
return [domains, total];
}
export const useAuthors = () => {
const [authors, setAuthors] = useState(null);
const [total, setTotal] = useState(null);
const _fetchAuthors = async () => {
setAuthors(null);
setTotal(0);
try {
let [authors, total] = await fetchAuthors();
setAuthors(authors);
setTotal(total);
} catch (e) {
setAuthors(undefined);
setTotal(0);
}
}
useEffect(() => {
_fetchAuthors();
}, []);
return [authors, total];
}
export const useSubjects = () => {
const [subjects, setSubjects] = useState(null);
const [total, setTotal] = useState(null);
const _fetchSubjects = async () => {
setSubjects(null);
setTotal(0);
try {
let [subjects, total] = await fetchSubjects();
setSubjects(subjects);
setTotal(total);
} catch (e) {
setSubjects(undefined);
setTotal(0);
}
}
useEffect(() => {
_fetchSubjects();
}, []);
return [subjects, total];
}
export const useDoc = (fileGroupId) => {
const [docFile, setDocFile] = useState(null);
const [doc, setDoc] = useState(null);
const fetchDoc = async () => {
try {
const response = await getDocByFileGroup(fileGroupId);
if (response.ok) {
const data = await response.json();
setDocFile(data.file_group);
setDoc(data.doc);
} else {
setDocFile(undefined);
}
} catch (e) {
console.error(e);
setDocFile(undefined);
}
}
useEffect(() => {
fetchDoc();
}, [fileGroupId]);
return {docFile, doc, fetchDoc};
}
export const useDomain = (domainId) => {
const emptyDomain = {
domain: '',
@ -455,6 +534,62 @@ export const useSearchArchives = (defaultLimit) => {
}
}
export const useSearchDocs = (defaultLimit) => {
const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
const {searchParams, updateQuery} = React.useContext(QueryContext);
const searchStr = searchParams.get('q') || '';
const order = searchParams.get('order');
const activeTags = searchParams.getAll('tag');
const author = searchParams.get('author') || '';
const subject = searchParams.get('subject') || '';
const [docs, setDocs] = useState(null);
const [totalPages, setTotalPages] = useState(0);
const localSearchDocs = async () => {
setDocs(null);
setTotalPages(0);
try {
let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject);
setDocs(docs_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
console.error(e);
setDocs(undefined);
}
}
useEffect(() => {
localSearchDocs();
}, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
}
const setOrderBy = (value) => {
setPage(1);
updateQuery({order: value});
}
return {
docs,
limit,
setLimit,
offset,
order,
setOrderBy,
totalPages,
activePage,
setPage,
searchStr,
setSearchStr,
author,
subject,
fetchDocs: localSearchDocs,
}
}
export const useSearchVideos = (defaultLimit, channelId, order_by) => {
const {searchParams, updateQuery} = React.useContext(QueryContext);
const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);

View file

@ -95,6 +95,7 @@ sysstat
tar
tcpdump
tmux
unrar-free
unzip
vim
vlc

View file

@ -3,7 +3,7 @@ ENV DOCKER=true
WORKDIR /opt/wrolpi
# Update dependencies for the services install.
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unzip
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip
RUN ffmpeg -version
# Install Deno runtime for yt-dlp

View file

@ -59,13 +59,6 @@
reverse_proxy api:8081
}
# API docs.
handle /docs/* {
reverse_proxy api:8081
}
handle /docs {
redir * /docs/ 301
}
# Custom MIME types not in Caddy's defaults.
@yaml path *.yaml *.yml

View file

@ -59,13 +59,6 @@
reverse_proxy 0.0.0.0:8081
}
# API docs.
handle /docs/* {
reverse_proxy 0.0.0.0:8081
}
handle /docs {
redir * /docs/ 301
}
# Custom MIME types not in Caddy's defaults.
@yaml path *.yaml *.yml

View file

@ -1,4 +1,5 @@
from .archive import api
from .docs import api
from .inventory import api
from .map import api
from .otp import api

188
modules/docs/__init__.py Normal file
View file

@ -0,0 +1,188 @@
import asyncio
import logging
from typing import Callable, List, Tuple
from sqlalchemy import or_
from sqlalchemy.orm import Session
from wrolpi.common import register_modeler, register_refresh_cleanup, \
truncate_object_bytes, split_lines_by_length
from wrolpi.db import get_db_session
from wrolpi.files.indexers import Indexer
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
from .extractors import extract_metadata
from .lib import get_or_create_subject_collection, get_or_create_author_collection, is_valid_author, split_authors, \
normalize_author, normalize_subject, split_subjects, is_valid_subject, discover_calibre_cover
from .models import Doc, DOC_MIMETYPES, COMIC_BOOK_SUFFIXES, EPUB_MIMETYPE, MOBI_MIMETYPE
logger = logging.getLogger(__name__)
DOC_PROCESSING_LIMIT = 10
@register_modeler
async def doc_modeler(progress_callback: Callable[[int], None] = None):
"""Searches for doc files (epub, mobi, pdf, docx, doc, odt, cbz, cbr) and models them."""
total_processed = 0
while True:
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup, Doc).filter(
or_(
*[FileGroup.mimetype == mt for mt in DOC_MIMETYPES],
*[FileGroup.primary_path.like(f'%{suffix}') for suffix in COMIC_BOOK_SUFFIXES],
),
FileGroup.indexed == False,
).outerjoin(Doc, Doc.file_group_id == FileGroup.id) \
.limit(DOC_PROCESSING_LIMIT)
file_groups: List[Tuple[FileGroup, Doc]] = list(file_groups)
processed = 0
for file_group, doc in file_groups:
processed += 1
try:
if PYTEST:
session.expire(file_group)
doc = _model_doc(file_group, session)
session.add(doc)
file_group.model = Doc.__tablename__
file_group.indexed = True
except Exception as e:
logger.error(f'Failed to index doc {file_group}', exc_info=e)
file_group.indexed = True # Don't retry on failure.
if PYTEST:
raise
session.commit()
total_processed += len(file_groups)
if progress_callback and len(file_groups) > 0:
progress_callback(total_processed)
if processed < DOC_PROCESSING_LIMIT:
break
await asyncio.sleep(0)
def _model_doc(file_group: FileGroup, session: Session) -> Doc:
"""Creates a Doc model from a FileGroup. Extracts metadata and creates collections."""
doc = session.query(Doc).filter_by(file_group_id=file_group.id).one_or_none()
if not doc:
doc = Doc(file_group=file_group)
session.add(doc)
metadata = extract_metadata(file_group)
if metadata.title:
file_group.title = file_group.a_text = metadata.title
if metadata.author:
file_group.author = file_group.b_text = metadata.author
if metadata.text:
text = truncate_object_bytes(metadata.text, FILE_MAX_TEXT_SIZE)
text = split_lines_by_length(text)
file_group.d_text = text
# Set c_text to filename for search.
file_title = Indexer.get_title(file_group.primary_path)
file_group.c_text = file_title
if not file_group.title:
file_group.title = file_title
if metadata.published_date:
file_group.published_datetime = metadata.published_date
# Update Doc-specific fields.
doc.publisher = metadata.publisher
doc.language = metadata.language
doc.page_count = metadata.page_count
doc.subject = metadata.subject
doc.description = metadata.description
doc.size = file_group.primary_path.stat().st_size if file_group.primary_path.exists() else None
# Handle cover.
_handle_doc_cover(doc, file_group, metadata)
# Store data dict.
data = file_group.data or {}
if metadata.title:
data['title'] = metadata.title
if metadata.author:
data['author'] = metadata.author
# Store primary doc path as relative filename.
data['doc_path'] = file_group.primary_path.name
file_group.data = data
# Auto-create collections.
_auto_create_collections(session, file_group, metadata)
doc.flush()
return doc
def _handle_doc_cover(doc: Doc, file_group: FileGroup, metadata):
"""Handle cover extraction and storage for a doc."""
# Check for Calibre cover first (epub only) — must happen before poster check
# since Calibre cover has a different stem ("cover.jpg" not matching epub stem).
if file_group.mimetype and file_group.mimetype.startswith('application/epub'):
if calibre_cover := discover_calibre_cover(file_group.primary_path):
doc._hide_calibre_files(file_group)
file_group.append_files(calibre_cover)
file_group.data = file_group.data or {}
file_group.data['cover_path'] = calibre_cover.name
return
# Use existing poster files if present.
poster_files = file_group.my_poster_files()
if poster_files:
file_group.data = file_group.data or {}
file_group.data['cover_path'] = poster_files[0]['path'].name
return
# Use cover from metadata extraction (epub internal cover, PDF first page, etc.)
cover_bytes = metadata.cover_bytes
if cover_bytes:
cover_path = file_group.primary_path.with_suffix('.jpeg')
cover_path.write_bytes(cover_bytes)
file_group.append_files(cover_path)
file_group.data = file_group.data or {}
file_group.data['cover_path'] = cover_path.name
def _auto_create_collections(session: Session, file_group: FileGroup, metadata):
"""Auto-create author/subject collections and link the file_group."""
if metadata.author:
for author in split_authors(metadata.author):
author = normalize_author(author)
if author and is_valid_author(author):
col = get_or_create_author_collection(session, author)
if col:
col.add_file_group(file_group, session=session)
if metadata.subject:
for subject in split_subjects(metadata.subject):
subject = normalize_subject(subject)
if subject and is_valid_subject(subject):
col = get_or_create_subject_collection(session, subject)
if col:
col.add_file_group(file_group, session=session)
@register_refresh_cleanup
def doc_cleanup(test_directory=None):
"""Remove empty author/subject collections after refresh."""
from sqlalchemy import func
from wrolpi.collections.models import Collection, CollectionItem
with get_db_session(commit=True) as session:
empty_collections = session.query(Collection).filter(
Collection.kind.in_(['author', 'subject']),
).outerjoin(CollectionItem).group_by(Collection.id).having(
func.count(CollectionItem.id) == 0,
).all()
for collection in empty_collections:
logger.info(f'Removing empty {collection.kind} collection: {collection.name}')
session.delete(collection)

85
modules/docs/api.py Normal file
View file

@ -0,0 +1,85 @@
from http import HTTPStatus
from sanic import response, Request, Blueprint
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger, wrol_mode_check, api_param_limiter
from wrolpi.errors import ValidationError
from wrolpi.schema import JSONErrorResponse
from . import schema
from .lib import get_statistics, _doc_response, _get_doc, _get_doc_by_file_group, _delete_docs, _search_docs
NAME = 'docs'
docs_bp = Blueprint('Docs', '/api/docs')
logger = logger.getChild(__name__)
@docs_bp.get('/statistics')
@openapi.response(HTTPStatus.OK, schema.DocStatisticsResponse)
async def statistics(_: Request):
ret = get_statistics()
return json_response(ret, HTTPStatus.OK)
@docs_bp.get('/<doc_id:int>')
@openapi.description('Get a doc by doc ID')
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
async def get_doc(request: Request, doc_id: int):
session = request.ctx.session
doc = _get_doc(session, doc_id)
return json_response(_doc_response(doc))
@docs_bp.get('/view/<file_group_id:int>')
@openapi.description('Get a doc by file_group ID')
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
async def get_doc_by_file_group(request: Request, file_group_id: int):
session = request.ctx.session
doc = _get_doc_by_file_group(session, file_group_id)
return json_response(_doc_response(doc))
@docs_bp.delete('/<doc_ids:int>', name='doc_delete_one')
@docs_bp.delete('/<doc_ids:[0-9,]+>', name='doc_delete_many')
@openapi.description('Delete Docs')
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@wrol_mode_check
async def delete_docs(_: Request, doc_ids: str):
try:
doc_ids = [int(i) for i in str(doc_ids).split(',')]
except ValueError:
raise ValidationError('Could not parse doc ids')
_delete_docs(*doc_ids)
return response.empty()
doc_limit_limiter = api_param_limiter(100)
@docs_bp.post('/search')
@openapi.definition(
summary='Search docs with filtering',
body=schema.DocSearchRequest,
)
@openapi.response(HTTPStatus.OK, schema.DocSearchResponse)
@validate(schema.DocSearchRequest)
async def search_docs(_: Request, body: schema.DocSearchRequest):
limit = doc_limit_limiter(body.limit)
offset = body.offset or 0
file_groups, total = _search_docs(
search_str=body.search_str,
author=body.author,
subject=body.subject,
language=body.language,
mimetype=body.mimetype,
limit=limit,
offset=offset,
order_by=body.order_by,
tag_names=body.tag_names,
)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)

26
modules/docs/conftest.py Normal file
View file

@ -0,0 +1,26 @@
import pytest
from modules.docs.models import Doc
@pytest.fixture
def doc_factory(test_session, test_directory):
"""Factory for creating Doc records for testing."""
count = 0
def factory(file_group=None, **kwargs):
nonlocal count
count += 1
if file_group is None:
from wrolpi.files.models import FileGroup
path = test_directory / f'test_doc_{count}.pdf'
path.write_bytes(b'%PDF-1.4 test')
file_group = FileGroup.from_paths(test_session, path)
test_session.flush()
doc = Doc(file_group=file_group, **kwargs)
test_session.add(doc)
test_session.flush()
return doc
return factory

428
modules/docs/extractors.py Normal file
View file

@ -0,0 +1,428 @@
import dataclasses
import datetime
import logging
import os
import pathlib
import zipfile
import pytz
from wrolpi import dates
from wrolpi.common import truncate_generator_bytes, truncate_object_bytes, get_html_soup
from wrolpi.files.indexers import Indexer
from wrolpi.files.models import FileGroup
from wrolpi.vars import FILE_MAX_PDF_SIZE, FILE_MAX_TEXT_SIZE
try:
import ebooklib
from ebooklib import epub
except ImportError:
ebooklib = None
epub = None
try:
import pymupdf
except ImportError:
pymupdf = None
try:
import docx
except ImportError:
docx = None
logger = logging.getLogger(__name__)
@dataclasses.dataclass
class DocMetadata:
title: str = None
author: str = None
publisher: str = None
published_date: datetime.datetime = None
page_count: int = None
language: str = None
subject: str = None
description: str = None
cover_bytes: bytes = None
text: str = None
def extract_metadata(file_group: FileGroup) -> DocMetadata:
"""Extract metadata from a FileGroup based on its mimetype."""
mimetype = file_group.mimetype or ''
path = file_group.primary_path
if mimetype.startswith('application/epub'):
return extract_epub(path)
elif mimetype == 'application/x-mobipocket-ebook':
# MOBI files can't be easily parsed for metadata.
return DocMetadata(title=Indexer.get_title(path))
elif mimetype == 'application/pdf':
return extract_pdf(path)
elif mimetype == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
return extract_docx(path)
elif mimetype == 'application/msword':
return extract_doc(path)
elif mimetype == 'application/vnd.oasis.opendocument.text':
return extract_odt(path)
elif mimetype in ('application/vnd.comicbook+zip', 'application/x-cbz',
'application/vnd.comicbook-rar', 'application/x-cbr'):
return extract_cbz(path)
elif path.suffix.lower() in ('.cbz', '.cbr', '.cbt', '.cb7'):
return extract_cbz(path)
return DocMetadata(title=Indexer.get_title(path))
def _extract_text_from_html(html: str) -> str:
soup = get_html_soup(html)
return soup.get_text()
def extract_epub(path: pathlib.Path) -> DocMetadata:
"""Extract metadata from an EPUB file using ebooklib."""
if ebooklib is None or epub is None:
raise ValueError('ebooklib is not installed')
metadata = DocMetadata()
try:
book = epub.read_epub(path, options=dict(ignore_ncx=True))
except Exception as e:
logger.error(f'Failed to read epub {path}', exc_info=e)
metadata.title = Indexer.get_title(path)
return metadata
for key, value in book.metadata.items():
if 'title' in value:
metadata.title = metadata.title or value['title'][0][0]
if 'creator' in value:
metadata.author = metadata.author or value['creator'][0][0]
if 'publisher' in value:
metadata.publisher = metadata.publisher or value['publisher'][0][0]
if 'language' in value:
metadata.language = metadata.language or value['language'][0][0]
if 'description' in value:
metadata.description = metadata.description or value['description'][0][0]
if 'subject' in value:
metadata.subject = metadata.subject or value['subject'][0][0]
# Extract text.
text_parts = []
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
text_parts.append(_extract_text_from_html(doc.content))
metadata.text = ''.join(text_parts)
# Extract cover.
try:
for doc in book.get_items_of_type(ebooklib.ITEM_COVER):
metadata.cover_bytes = doc.content
break
if not metadata.cover_bytes:
for doc in book.get_items_of_type(ebooklib.ITEM_IMAGE):
metadata.cover_bytes = doc.content
break
except Exception as e:
logger.warning(f'Failed to extract cover from {path}', exc_info=e)
return metadata
def _parse_pdf_date(date_str: str):
"""Parse PDF date format 'D:YYYYMMDDHHmmSS+HH'mm'' to datetime."""
if not date_str:
return None
try:
parsed = dates.strpdate(date_str)
if parsed and not parsed.tzinfo:
parsed = parsed.astimezone(pytz.UTC)
return parsed
except Exception:
return None
def extract_pdf(path: pathlib.Path) -> DocMetadata:
"""Extract metadata from a PDF using pymupdf."""
if pymupdf is None:
logger.error(f'Cannot extract PDF metadata: pymupdf is not installed')
return DocMetadata(title=Indexer.get_title(path))
metadata = DocMetadata()
doc = None
try:
doc = pymupdf.open(path)
pdf_meta = doc.metadata or {}
if (title := pdf_meta.get('title')) and title.lower() != 'unknown':
metadata.title = title
if (author := pdf_meta.get('author')) and author.lower() != 'unknown':
metadata.author = author
if subject := pdf_meta.get('subject'):
metadata.subject = subject
metadata.page_count = doc.page_count
if creation_date := pdf_meta.get('creationDate'):
metadata.published_date = _parse_pdf_date(creation_date)
# Extract text.
if path.stat().st_size <= FILE_MAX_PDF_SIZE:
try:
words = []
for page in doc:
text = page.get_text()
text = text.replace('\x00', '\uFFFD').strip()
if text:
words.append(text)
metadata.text = '\n'.join(
truncate_generator_bytes(iter(words), FILE_MAX_TEXT_SIZE))
except Exception as e:
logger.error(f'Failed to extract PDF text: {path}', exc_info=e)
# Extract first page as cover thumbnail.
try:
if doc.page_count > 0:
page = doc[0]
pix = page.get_pixmap(matrix=pymupdf.Matrix(0.5, 0.5))
metadata.cover_bytes = pix.tobytes('jpeg')
except Exception as e:
logger.warning(f'Failed to render PDF cover: {path}', exc_info=e)
except Exception as e:
logger.error(f'Failed to extract PDF metadata: {path}', exc_info=e)
metadata.title = Indexer.get_title(path)
finally:
if doc:
doc.close()
return metadata
def extract_docx(path: pathlib.Path) -> DocMetadata:
"""Extract metadata from a DOCX file."""
if docx is None:
return DocMetadata(title=Indexer.get_title(path))
metadata = DocMetadata()
try:
document = docx.Document(str(path))
props = document.core_properties
metadata.title = props.title or None
metadata.author = props.author or None
metadata.subject = props.subject or None
metadata.description = props.comments or None
metadata.language = props.language or None
if props.created:
metadata.published_date = props.created
# Extract text.
text_parts = []
for para in document.paragraphs:
if para.text:
text_parts.append(para.text)
metadata.text = truncate_object_bytes(' '.join(text_parts), FILE_MAX_TEXT_SIZE)
except Exception as e:
logger.error(f'Failed to extract docx metadata: {path}', exc_info=e)
metadata.title = Indexer.get_title(path)
return metadata
def extract_doc(path: pathlib.Path) -> DocMetadata:
"""Extract text from old .doc files using catdoc/textutil."""
import subprocess
from wrolpi.cmd import CATDOC_PATH, TEXTUTIL_PATH
metadata = DocMetadata()
metadata.title = Indexer.get_title(path)
try:
if CATDOC_PATH:
cmd = (CATDOC_PATH, str(path.absolute()))
proc = subprocess.run(cmd, capture_output=True)
metadata.text = proc.stdout.decode()
elif TEXTUTIL_PATH:
cmd = (TEXTUTIL_PATH, '-stdout', '-cat', 'txt', str(path.absolute()))
proc = subprocess.run(cmd, capture_output=True)
metadata.text = proc.stdout.decode()
if metadata.text:
metadata.text = truncate_object_bytes(metadata.text, FILE_MAX_TEXT_SIZE)
except Exception as e:
logger.error(f'Failed to extract doc text: {path}', exc_info=e)
return metadata
def extract_odt(path: pathlib.Path) -> DocMetadata:
"""Extract metadata from an ODT file by parsing meta.xml inside the zip."""
metadata = DocMetadata()
metadata.title = Indexer.get_title(path)
try:
with zipfile.ZipFile(path, 'r') as zf:
if 'meta.xml' in zf.namelist():
meta_xml = zf.read('meta.xml').decode('utf-8')
import xml.etree.ElementTree as ET
root = ET.fromstring(meta_xml)
ns = {
'office': 'urn:oasis:names:tc:opendocument:xmlns:office:1.0',
'dc': 'http://purl.org/dc/elements/1.1/',
'meta': 'urn:oasis:names:tc:opendocument:xmlns:meta:1.0',
}
office_meta = root.find('.//office:meta', ns)
if office_meta is not None:
title_el = office_meta.find('dc:title', ns)
if title_el is not None and title_el.text:
metadata.title = title_el.text
creator_el = office_meta.find('dc:creator', ns)
if creator_el is not None and creator_el.text:
metadata.author = creator_el.text
subject_el = office_meta.find('dc:subject', ns)
if subject_el is not None and subject_el.text:
metadata.subject = subject_el.text
desc_el = office_meta.find('dc:description', ns)
if desc_el is not None and desc_el.text:
metadata.description = desc_el.text
lang_el = office_meta.find('dc:language', ns)
if lang_el is not None and lang_el.text:
metadata.language = lang_el.text
# Extract text from content.xml.
if 'content.xml' in zf.namelist():
content_xml = zf.read('content.xml').decode('utf-8')
import xml.etree.ElementTree as ET
root = ET.fromstring(content_xml)
# Get all text content.
texts = []
for elem in root.iter():
if elem.text:
texts.append(elem.text)
if elem.tail:
texts.append(elem.tail)
metadata.text = truncate_object_bytes(' '.join(texts), FILE_MAX_TEXT_SIZE)
except Exception as e:
logger.error(f'Failed to extract ODT metadata: {path}', exc_info=e)
return metadata
def _list_comic_archive(path: pathlib.Path):
"""Return (file_names, read_func) for a comic book archive.
read_func(name) returns the bytes of the named member."""
suffix = path.suffix.lower()
if suffix in ('.cbz', '.zip'):
zf = zipfile.ZipFile(path, 'r')
return zf.namelist(), zf.read, zf
elif suffix in ('.cbt',):
import tarfile as _tarfile
tf = _tarfile.open(path, 'r:*')
names = [m.name for m in tf.getmembers() if not m.isdir()]
def read_tar(name):
f = tf.extractfile(name)
return f.read() if f else b''
return names, read_tar, tf
elif suffix in ('.cbr',):
import rarfile
rf = rarfile.RarFile(path, 'r')
return rf.namelist(), rf.read, rf
elif suffix in ('.cb7',):
import py7zr
import tempfile
sz = py7zr.SevenZipFile(path, 'r')
tmpdir = tempfile.mkdtemp()
sz.extractall(tmpdir)
sz.close()
names = []
for root, dirs, files in os.walk(tmpdir):
for f in files:
rel = os.path.relpath(os.path.join(root, f), tmpdir)
names.append(rel)
def read_7z(name):
return (pathlib.Path(tmpdir) / name).read_bytes()
class Cleanup:
def close(self):
import shutil
shutil.rmtree(tmpdir, ignore_errors=True)
return names, read_7z, Cleanup()
return [], lambda n: b'', None
def extract_cbz(path: pathlib.Path) -> DocMetadata:
"""Extract metadata from comic book archives (CBZ, CBT, CBR, CB7)."""
metadata = DocMetadata()
metadata.title = Indexer.get_title(path)
try:
names, read_func, ctx = _list_comic_archive(path)
try:
# Look for ComicInfo.xml.
for name in names:
if name.lower().endswith('comicinfo.xml'):
try:
comic_xml = read_func(name).decode('utf-8')
import xml.etree.ElementTree as ET
root = ET.fromstring(comic_xml)
title_el = root.find('Title')
series_el = root.find('Series')
number_el = root.find('Number')
if series_el is not None and series_el.text:
# Use "Series #Number" as title if available.
title = series_el.text
if number_el is not None and number_el.text:
title = f'{title} #{number_el.text}'
metadata.title = title
elif title_el is not None and title_el.text:
metadata.title = title_el.text
writer_el = root.find('Writer')
if writer_el is not None and writer_el.text:
metadata.author = writer_el.text
summary_el = root.find('Summary')
if summary_el is not None and summary_el.text:
metadata.description = summary_el.text
page_count_el = root.find('PageCount')
if page_count_el is not None and page_count_el.text:
metadata.page_count = int(page_count_el.text)
publisher_el = root.find('Publisher')
if publisher_el is not None and publisher_el.text:
metadata.publisher = publisher_el.text
lang_el = root.find('LanguageISO')
if lang_el is not None and lang_el.text:
metadata.language = lang_el.text
genre_el = root.find('Genre')
if genre_el is not None and genre_el.text:
metadata.subject = genre_el.text
# Build published_date from Year/Month/Day.
year_el = root.find('Year')
if year_el is not None and year_el.text:
year = int(year_el.text)
month_el = root.find('Month')
day_el = root.find('Day')
month = int(month_el.text) if month_el is not None and month_el.text else 1
day = int(day_el.text) if day_el is not None and day_el.text else 1
metadata.published_date = datetime.datetime(year, month, day, tzinfo=pytz.UTC)
except Exception as e:
logger.warning(f'Failed to parse ComicInfo.xml: {path}', exc_info=e)
break
# Extract first image as cover.
image_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.webp')
image_names = sorted([n for n in names
if any(n.lower().endswith(ext) for ext in image_extensions)])
if image_names:
metadata.cover_bytes = read_func(image_names[0])
finally:
if ctx:
ctx.close()
except Exception as e:
logger.error(f'Failed to extract comic metadata: {path}', exc_info=e)
return metadata

417
modules/docs/lib.py Normal file
View file

@ -0,0 +1,417 @@
import html
import logging
import pathlib
import re
from typing import List, Optional
from sqlalchemy import asc, desc, func, nullslast
from sqlalchemy.orm import Session
from wrolpi.collections.models import Collection
from wrolpi.collections.types import collection_type_registry
from wrolpi.db import get_db_session
from wrolpi.errors import ValidationError
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
logger = logging.getLogger(__name__)
# Register author and subject collection types.
collection_type_registry.register(
'author',
lambda n: bool(n and n.strip()),
'Non-empty author name'
)
collection_type_registry.register(
'subject',
lambda n: bool(n and n.strip()),
'Non-empty subject name'
)
# Junk author patterns to reject.
JUNK_AUTHORS = {
'administrator', 'admin', 'unknown', 'preferred customer', 'user',
'owner', 'default', 'none', 'n/a', 'na', 'test', 'root',
}
JUNK_SUBJECTS = {
'unknown', 'n/a', 'na', 'none', 'test', 'default',
}
# Regex for URL-like strings.
URL_PATTERN = re.compile(r'https?://|www\.|\.com|\.org|\.net')
# Matches year ranges like 1483-1546, -1938, 1564-1616.
YEAR_RANGE_PATTERN = re.compile(r'^-?\d{0,4}-?\d{4}$')
# Matches bracket suffixes like [from old catalog].
BRACKET_SUFFIX_PATTERN = re.compile(r'\s*\]?\s*\[.*$')
# Matches trailing birth-death years like 1843-1920 at end of string.
TRAILING_YEARS_PATTERN = re.compile(r'\s+\d{4}-\d{4}$')
# Matches pure numeric/code patterns: 00110001, 062447, 105642-01, 01/2004.
NUMERIC_CODE_PATTERN = re.compile(r'^\d[\d\-/]*$')
# Matches date strings like "14 June 2006".
DATE_STRING_PATTERN = re.compile(r'^\d{1,2}\s+\w+\s+\d{4}$')
# Matches HTML entities like &bull; or &amp;
HTML_ENTITY_PATTERN = re.compile(r'&\w+;')
# Matches initials pattern like J.K. (used to avoid stripping trailing period).
INITIALS_PATTERN = re.compile(r'\.\w\.')
def normalize_author(author: str) -> Optional[str]:
"""Clean an author string before validation."""
if not author or not author.strip():
return None
author = author.strip()
# Remove bracket suffixes like [from old catalog].
author = BRACKET_SUFFIX_PATTERN.sub('', author)
# Strip surrounding quotes.
if len(author) >= 2 and author[0] == '"' and author[-1] == '"':
author = author[1:-1]
# Also strip leading quote left after bracket removal.
if author.startswith('"'):
author = author[1:]
# Remove surrounding parentheses.
if len(author) >= 2 and author[0] == '(' and author[-1] == ')':
author = author[1:-1]
# Remove trailing birth-death years.
author = TRAILING_YEARS_PATTERN.sub('', author)
# Strip trailing period, but not if it's part of initials (e.g. J.K.).
if author.endswith('.') and not INITIALS_PATTERN.search(author):
author = author[:-1]
# Strip trailing comma.
author = author.rstrip(',')
# Title-case for consistent casing.
author = author.title()
author = author.strip()
return author if author else None
def normalize_subject(subject: str) -> Optional[str]:
"""Clean a subject string before validation."""
if not subject or not subject.strip():
return None
subject = subject.strip()
# Decode HTML entities.
subject = html.unescape(subject)
# Title-case for consistent casing.
subject = subject.title()
# Truncate to 150 chars.
if len(subject) > 150:
subject = subject[:150]
subject = subject.strip()
return subject if subject else None
# Matches pure digits.
PURE_DIGITS_PATTERN = re.compile(r'^\d+$')
# Matches any letter character.
HAS_LETTER_PATTERN = re.compile(r'[a-zA-Z]')
def is_valid_author(author: str) -> bool:
"""Check if an author string looks like a real author name."""
if not author or not author.strip():
return False
author = author.strip()
if len(author) > 100:
return False
if len(author) < 2:
return False
if author.lower() in JUNK_AUTHORS:
return False
if URL_PATTERN.search(author):
return False
# Reject pure year ranges and numbers.
if YEAR_RANGE_PATTERN.match(author):
return False
if PURE_DIGITS_PATTERN.match(author):
return False
# Reject strings with no letter characters.
if not HAS_LETTER_PATTERN.search(author):
return False
return True
def is_valid_subject(subject: str) -> bool:
"""Check if a subject string looks like a real subject."""
if not subject or not subject.strip():
return False
subject = subject.strip()
if len(subject) < 2:
return False
if len(subject) > 150:
return False
if subject.lower() in JUNK_SUBJECTS:
return False
if URL_PATTERN.search(subject):
return False
# Reject pure numeric/code patterns.
if NUMERIC_CODE_PATTERN.match(subject):
return False
# Reject date strings.
if DATE_STRING_PATTERN.match(subject):
return False
# Reject strings with HTML entities.
if HTML_ENTITY_PATTERN.search(subject):
return False
return True
# Splits multiple authors on common separators.
AUTHOR_SPLIT_PATTERN = re.compile(r'[;&]|,\s*(?:and|&)\s*|,\s+')
def split_authors(author_str: str) -> List[str]:
"""Split an author string that may contain multiple authors."""
if not author_str:
return []
# Split on common separators.
authors = AUTHOR_SPLIT_PATTERN.split(author_str)
return [a.strip() for a in authors if a.strip()]
# Splits multiple subjects on semicolons and bare commas.
SUBJECT_SPLIT_PATTERN = re.compile(r';|,(?!\s)')
def split_subjects(subject_str: str) -> List[str]:
"""Split a subject string that may contain multiple subjects."""
if not subject_str:
return []
# Split on semicolons and bare commas (comma NOT followed by a space).
subjects = SUBJECT_SPLIT_PATTERN.split(subject_str)
return [s.strip() for s in subjects if s.strip()]
def get_or_create_author_collection(session: Session, author_name: str) -> Optional[Collection]:
"""Find or create a Collection for an author."""
if not author_name or not author_name.strip():
return None
author_name = author_name.strip()
collection = session.query(Collection).filter(
func.lower(Collection.name) == author_name.lower(),
Collection.kind == 'author',
).first()
if not collection:
collection = Collection(name=author_name, kind='author')
session.add(collection)
session.flush()
logger.debug(f'Created author collection: {author_name}')
return collection
def get_or_create_subject_collection(session: Session, subject_name: str) -> Optional[Collection]:
"""Find or create a Collection for a subject."""
if not subject_name or not subject_name.strip():
return None
subject_name = subject_name.strip()
collection = session.query(Collection).filter(
func.lower(Collection.name) == subject_name.lower(),
Collection.kind == 'subject',
).first()
if not collection:
collection = Collection(name=subject_name, kind='subject')
session.add(collection)
session.flush()
logger.debug(f'Created subject collection: {subject_name}')
return collection
async def search_authors_by_name(session: Session, name: str, limit: int = 5) -> List[dict]:
"""Search for author collections by partial name."""
collections = session.query(Collection) \
.filter(Collection.kind == 'author') \
.filter(Collection.name.ilike(f'%{name}%')) \
.order_by(asc(Collection.name)) \
.limit(limit) \
.all()
return [{'id': c.id, 'name': c.name} for c in collections]
async def search_subjects_by_name(session: Session, name: str, limit: int = 5) -> List[dict]:
"""Search for subject collections by partial name."""
collections = session.query(Collection) \
.filter(Collection.kind == 'subject') \
.filter(Collection.name.ilike(f'%{name}%')) \
.order_by(asc(Collection.name)) \
.limit(limit) \
.all()
return [{'id': c.id, 'name': c.name} for c in collections]
def discover_calibre_cover(ebook_path: pathlib.Path):
"""Calibre puts a cover file in a directory which contains only one ebook (but multiple formats of the ebook),
this function will return the path of the cover if the ebook_path is in such a situation."""
if not ebook_path.is_file():
raise ValueError(f'Invalid ebook path {ebook_path}')
paths = list(ebook_path.parent.iterdir())
ebook_path_stem, _ = split_path_stem_and_suffix(ebook_path)
cover = None
for path in paths:
stem, suffix = split_path_stem_and_suffix(path)
if path.is_dir():
# We don't care about sub directories.
continue
if stem == ebook_path_stem:
# This is one of the formats of the ebook.
continue
if path.name == 'metadata.opf':
# Metadata, this is expected in a Calibre book directory.
continue
if stem == 'cover' and get_mimetype(path).startswith('image/'):
# Discovered the cover.
cover = path
continue
# Some file that does not match the ebook stem, metadata, or cover. This must not be a Calibre book directory.
return None
return cover
def get_statistics() -> dict:
"""Get doc statistics."""
from .models import Doc
from wrolpi.files.models import FileGroup
with get_db_session() as session:
total = session.query(func.count(Doc.id)).scalar() or 0
total_size = session.query(func.sum(Doc.size)).scalar() or 0
epub_count = session.query(func.count(Doc.id)).join(FileGroup).filter(
FileGroup.mimetype.startswith('application/epub')).scalar() or 0
pdf_count = session.query(func.count(Doc.id)).join(FileGroup).filter(
FileGroup.mimetype == 'application/pdf').scalar() or 0
author_count = session.query(func.count(Collection.id)).filter(
Collection.kind == 'author').scalar() or 0
subject_count = session.query(func.count(Collection.id)).filter(
Collection.kind == 'subject').scalar() or 0
return dict(statistics=dict(
doc_count=total,
epub_count=epub_count,
pdf_count=pdf_count,
other_count=total - epub_count - pdf_count,
total_size=total_size,
author_count=author_count,
subject_count=subject_count,
))
def _doc_response(doc) -> dict:
"""Build a JSON response dict for a Doc."""
file_group = doc.file_group.__json__()
return {
'file_group': file_group,
'doc': {
'id': doc.id,
'publisher': doc.publisher,
'language': doc.language,
'page_count': doc.page_count,
'subject': doc.subject,
'description': doc.description,
'size': doc.size,
}
}
def _get_doc(session, doc_id: int):
from .models import Doc
doc = session.query(Doc).filter_by(id=doc_id).one_or_none()
if not doc:
raise ValidationError(f'Doc {doc_id} not found')
return doc
def _get_doc_by_file_group(session, file_group_id: int):
from .models import Doc
doc = session.query(Doc).filter_by(file_group_id=file_group_id).one_or_none()
if not doc:
raise ValidationError(f'Doc with file_group_id {file_group_id} not found')
return doc
def _delete_docs(*doc_ids):
from .models import Doc
with get_db_session(commit=True) as session:
for doc_id in doc_ids:
doc = session.query(Doc).filter_by(id=doc_id).one_or_none()
if doc:
session.delete(doc)
def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
limit=20, offset=0, order_by='published_datetime', tag_names=None):
from .models import Doc
from wrolpi.files.models import FileGroup
with get_db_session() as session:
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
if search_str:
query = query.filter(FileGroup.textsearch.match(search_str))
if author:
query = query.filter(FileGroup.b_text.ilike(f'%{author}%'))
if subject:
query = query.filter(Doc.subject.ilike(f'%{subject}%'))
if language:
query = query.filter(Doc.language == language)
if mimetype:
query = query.filter(FileGroup.mimetype.startswith(mimetype))
if tag_names:
from wrolpi.tags import Tag, TagFile
tagged_fg_ids = session.query(TagFile.file_group_id) \
.join(Tag, Tag.id == TagFile.tag_id) \
.filter(Tag.name.in_(tag_names)) \
.subquery()
query = query.filter(FileGroup.id.in_(tagged_fg_ids))
total = query.count()
# Ordering.
if order_by == 'rank' and search_str:
query = query.order_by(
desc(func.ts_rank(FileGroup.textsearch, func.websearch_to_tsquery(search_str))),
desc(FileGroup.id),
)
elif order_by == 'published_datetime':
query = query.order_by(nullslast(desc(FileGroup.published_datetime)), desc(FileGroup.id))
elif order_by == 'size':
query = query.order_by(nullslast(desc(Doc.size)))
elif order_by == 'title':
query = query.order_by(nullslast(asc(FileGroup.a_text)))
else:
query = query.order_by(desc(FileGroup.id))
file_groups = query.offset(offset).limit(limit).all()
results = [fg.__json__() for fg in file_groups]
return results, total

122
modules/docs/models.py Normal file
View file

@ -0,0 +1,122 @@
import pathlib
from typing import Optional
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_
from sqlalchemy.orm import relationship, Session
from wrolpi.common import ModelHelper, Base
from wrolpi.db import get_db_session
from wrolpi.files.models import FileGroup
EPUB_MIMETYPE = 'application/epub'
MOBI_MIMETYPE = 'application/x-mobipocket-ebook'
DOC_MIMETYPES = (
'application/epub+zip',
'application/x-mobipocket-ebook',
'application/pdf',
'application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.oasis.opendocument.text',
'application/vnd.comicbook-rar',
'application/vnd.comicbook+zip',
'application/x-cbz',
'application/x-cbr',
'application/x-cbt',
'application/x-cb7',
)
# Mimetypes that are considered ebook formats.
EBOOK_MIMETYPES = (
EPUB_MIMETYPE,
MOBI_MIMETYPE,
)
COMIC_BOOK_SUFFIXES = ('.cbz', '.cbr', '.cbt', '.cb7')
def mimetype_is_doc(mimetype: str) -> bool:
"""Check if a mimetype is a doc type that the Doc model handles."""
return any(mimetype.startswith(mt) for mt in DOC_MIMETYPES)
class Doc(ModelHelper, Base):
__tablename__ = 'doc'
__table_args__ = (
Index('doc_size_idx', 'size'),
)
id = Column(Integer, primary_key=True)
size = Column(BigInteger)
publisher = Column(String)
language = Column(String)
page_count = Column(Integer)
subject = Column(String)
description = Column(Text)
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
file_group: FileGroup = relationship('FileGroup')
def __repr__(self):
path = str(self.file_group.primary_path) if self.file_group else 'None'
return f'<Doc id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
def __json__(self) -> dict:
d = self.file_group.__json__()
if d.get('data') and d['data'].get('doc_path'):
d['data']['doc_path'] = self.file_group.resolve_path(d['data']['doc_path'])
if d.get('data') and d['data'].get('cover_path'):
d['data']['cover_path'] = self.file_group.resolve_path(d['data']['cover_path'])
# Also handle legacy ebook_path key.
if d.get('data') and d['data'].get('ebook_path'):
d['data']['ebook_path'] = self.file_group.resolve_path(d['data']['ebook_path'])
return d
@staticmethod
def can_model(file_group: FileGroup) -> bool:
if file_group.mimetype and mimetype_is_doc(file_group.mimetype):
return True
if file_group.primary_path and file_group.primary_path.suffix.lower() in COMIC_BOOK_SUFFIXES:
return True
return False
@staticmethod
def do_model(session: Session, file_group: FileGroup) -> 'Doc':
from modules.docs import _model_doc
doc = _model_doc(file_group, session)
file_group.model = Doc.__tablename__
file_group.indexed = True
return doc
@property
def cover_file(self) -> Optional[dict]:
posters = self.file_group.my_poster_files()
if posters:
return posters[0]
return None
@property
def cover_path(self) -> Optional[pathlib.Path]:
cover = self.cover_file
if cover:
return cover['path']
return None
def _hide_calibre_files(self, file_group: FileGroup):
"""Delete FileGroups of the cover/metadata when this is a Calibre ebook directory."""
from modules.docs.lib import discover_calibre_cover
path = file_group.primary_path
if not discover_calibre_cover(path):
raise ValueError(f'Refusing to hide Calibre files when we do not have a Calibre ebook directory!')
metadata = path.parent / 'metadata.opf'
cover = path.parent / 'cover.jpg'
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup).filter(or_(
FileGroup.primary_path == str(metadata),
FileGroup.primary_path == str(cover),
))
for fg in file_groups:
session.delete(fg)

43
modules/docs/schema.py Normal file
View file

@ -0,0 +1,43 @@
import dataclasses
from typing import Optional
@dataclasses.dataclass
class DocSearchRequest:
search_str: Optional[str] = None
author: Optional[str] = None
subject: Optional[str] = None
language: Optional[str] = None
mimetype: Optional[str] = None
tag_names: Optional[list] = dataclasses.field(default_factory=list)
order_by: Optional[str] = 'published_datetime'
limit: int = 20
offset: int = 0
@dataclasses.dataclass
class DocSearchResponse:
file_groups: list = dataclasses.field(default_factory=list)
totals: dict = dataclasses.field(default_factory=dict)
@dataclasses.dataclass
class DocStatistics:
doc_count: int = 0
epub_count: int = 0
pdf_count: int = 0
other_count: int = 0
total_size: int = 0
author_count: int = 0
subject_count: int = 0
@dataclasses.dataclass
class DocStatisticsResponse:
statistics: dict = dataclasses.field(default_factory=dict)
@dataclasses.dataclass
class DocFileFormatRequest:
name: str = ''
model_tag: str = 'docs'

View file

View file

@ -0,0 +1,79 @@
"""Tests for the docs API."""
import json
from http import HTTPStatus
import pytest
from modules.docs.models import Doc
@pytest.mark.asyncio
async def test_docs_statistics(async_client, test_session, test_directory, example_epub, refresh_files):
"""Doc statistics endpoint returns correct counts."""
await refresh_files()
request, response = await async_client.get('/api/docs/statistics')
assert response.status_code == HTTPStatus.OK
stats = response.json['statistics']
assert stats['doc_count'] >= 1
assert stats['epub_count'] >= 1
@pytest.mark.asyncio
async def test_docs_search(async_client, test_session, test_directory, example_epub, refresh_files):
"""Docs search endpoint works."""
await refresh_files()
content = dict(search_str='WROLPi')
request, response = await async_client.post('/api/docs/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert len(response.json['file_groups']) >= 1
@pytest.mark.asyncio
async def test_get_doc(async_client, test_session, test_directory, example_epub, refresh_files):
"""Single doc endpoint returns the doc."""
await refresh_files()
doc = test_session.query(Doc).first()
assert doc
request, response = await async_client.get(f'/api/docs/{doc.id}')
assert response.status_code == HTTPStatus.OK
assert response.json['file_group']
@pytest.mark.asyncio
async def test_docs_search_by_tag(async_client, test_session, doc_factory, tag_factory):
"""Docs can be searched by tag."""
tag = await tag_factory()
doc = doc_factory()
doc.file_group.add_tag(test_session, tag.name)
test_session.commit()
# Search with matching tag returns the doc.
content = dict(tag_names=[tag.name])
request, response = await async_client.post('/api/docs/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['totals']['file_groups'] == 1
# Search with non-existent tag returns nothing.
content = dict(tag_names=['nonexistent'])
request, response = await async_client.post('/api/docs/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['totals']['file_groups'] == 0
@pytest.mark.asyncio
async def test_delete_doc(async_client, test_session, test_directory, example_epub, refresh_files):
"""Docs can be deleted."""
await refresh_files()
doc = test_session.query(Doc).first()
assert doc
doc_id = doc.id
request, response = await async_client.delete(f'/api/docs/{doc_id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Doc).filter_by(id=doc_id).one_or_none() is None

View file

@ -0,0 +1,277 @@
"""Tests for the doc_modeler."""
import shutil
from unittest.mock import patch
import pytest
from PIL import Image
from modules.docs import doc_modeler
from modules.docs.models import Doc
from wrolpi.dates import now
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.files.worker import file_worker
from wrolpi.test.common import assert_dict_contains
from wrolpi.vars import PROJECT_DIR
@pytest.mark.asyncio
async def test_doc_modeler_epub(async_client, test_session, test_directory, example_epub, example_mobi, refresh_files):
"""Doc modeler indexes epub files."""
await refresh_files()
doc: Doc = test_session.query(Doc).one()
assert doc.file_group.title == 'WROLPi Test Book'
assert doc.file_group.mimetype.startswith('application/epub')
assert doc.file_group.author == 'roland'
assert doc.file_group.a_text, 'Book title was not indexed'
assert doc.file_group.b_text, 'Book author was not indexed'
assert doc.file_group.d_text, 'Book text was not indexed'
assert doc.file_group.model == 'doc'
# Both ebooks are assumed to be the same book.
assert (epubs := doc.file_group.my_files('application/epub+zip')) and len(epubs) == 1
assert (mobis := doc.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1
# Cover was discovered.
assert len(doc.file_group.my_poster_files()) == 1
# A second refresh should not re-model the already-indexed doc.
with patch('modules.docs._model_doc', side_effect=AssertionError('_model_doc should not be called')):
await refresh_files()
@pytest.mark.asyncio
async def test_doc_modeler_pdf(async_client, test_session, test_directory, example_pdf, refresh_files):
"""Doc modeler indexes PDF files."""
await refresh_files()
doc: Doc = test_session.query(Doc).one()
assert doc.file_group.mimetype == 'application/pdf'
assert doc.file_group.model == 'doc'
assert doc.file_group.indexed is True
assert doc.size # Has a size.
@pytest.mark.asyncio
async def test_doc_do_model_sets_model(test_session, example_pdf):
"""Doc.do_model sets file_group.model so the FileGroup is recognized as a doc."""
files_lib._upsert_files([example_pdf], now())
file_group = test_session.query(FileGroup).one()
# Reset to simulate a newly discovered file.
file_group.model = None
file_group.indexed = False
test_session.commit()
file_group.do_model(test_session)
test_session.commit()
assert file_group.model == 'doc', 'do_model must set file_group.model'
assert file_group.indexed is True
@pytest.mark.asyncio
async def test_doc_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory):
"""Doc modeler processes more than one batch of files (limit=10)."""
ebook_dir = test_directory / 'ebooks'
ebook_dir.mkdir(parents=True)
num_docs = 15
for i in range(num_docs):
path = ebook_dir / f'test_ebook_{i:03d}.epub'
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', path)
# Create FileGroups.
for path in ebook_dir.iterdir():
fg = FileGroup.from_paths(test_session, path)
assert fg.mimetype.startswith('application/epub')
test_session.commit()
await doc_modeler()
doc_count = test_session.query(Doc).count()
assert doc_count == num_docs, \
f'doc_modeler should process ALL {num_docs} files, but only processed {doc_count}'
still_unindexed = test_session.query(FileGroup).filter(
FileGroup.indexed == False,
FileGroup.mimetype == 'application/epub+zip',
).count()
assert still_unindexed == 0
@pytest.mark.asyncio
async def test_doc_modeler_reindex_creates_collections(async_client, test_session, test_directory, example_epub,
refresh_files):
"""When a doc is re-indexed (indexed=false), author/subject collections should be re-created."""
from wrolpi.collections.models import Collection
from wrolpi.db import get_db_session
await refresh_files()
# Author collection was created during first modeling.
with get_db_session() as session:
doc = session.query(Doc).one()
assert doc.file_group.author == 'roland'
author_col = session.query(Collection).filter_by(kind='author', name='Roland').one_or_none()
assert author_col, 'Author collection should exist after first indexing'
# Simulate post-migration state: delete the collection and mark as unindexed.
with get_db_session(commit=True) as session:
author_col = session.query(Collection).filter_by(kind='author', name='Roland').one()
session.delete(author_col)
doc = session.query(Doc).one()
doc.file_group.indexed = False
# Verify collection is gone.
with get_db_session() as session:
assert session.query(Collection).filter_by(kind='author', name='Roland').one_or_none() is None
# Re-run the modeler.
await doc_modeler()
# Author collection should be re-created.
with get_db_session() as session:
author_col = session.query(Collection).filter_by(kind='author', name='Roland').one_or_none()
assert author_col, 'Author collection should be re-created when doc is re-indexed'
@pytest.mark.asyncio
async def test_doc_modeler_calibre_cover(test_session, async_client, test_directory, example_epub, example_mobi,
image_file, refresh_files):
"""Calibre cover near an ebook file is discovered."""
metadata = test_directory / 'metadata.opf'
metadata.touch()
cover_image = test_directory / 'cover.jpg'
shutil.move(image_file, cover_image)
await refresh_files()
doc: Doc = test_session.query(Doc).one()
assert doc.file_group.title == 'WROLPi Test Book'
assert doc.cover_path
assert {i['path'].name for i in doc.file_group.my_files()} == {'ebook example.epub', 'ebook example.mobi',
'cover.jpg'}
# Metadata and cover FileGroups were deleted.
assert test_session.query(FileGroup).count() == 1
@pytest.mark.asyncio
async def test_doc_modeler_normalizes_author(async_client, test_session, test_directory, example_epub, refresh_files):
"""Author names are normalized before creating collections (e.g. quotes stripped)."""
from wrolpi.collections.models import Collection
from wrolpi.db import get_db_session
# Patch extract_metadata to return a quoted author.
from modules.docs.extractors import DocMetadata
fake_metadata = DocMetadata(author='"Tony R. Kuphaldt"', title='Test Book')
with patch('modules.docs.extract_metadata', return_value=fake_metadata):
await refresh_files()
with get_db_session() as session:
# Collection should have normalized name (no quotes).
col = session.query(Collection).filter_by(kind='author', name='Tony R. Kuphaldt').one_or_none()
assert col, 'Author collection should have normalized name without quotes'
# No collection with the quoted name.
quoted = session.query(Collection).filter_by(kind='author', name='"Tony R. Kuphaldt"').one_or_none()
assert not quoted, 'Should not create collection with quoted author name'
@pytest.mark.asyncio
async def test_doc_modeler_validates_subject(async_client, test_session, test_directory, example_epub, refresh_files):
"""Junk subjects are rejected during modeling."""
from wrolpi.collections.models import Collection
from wrolpi.db import get_db_session
from modules.docs.extractors import DocMetadata
fake_metadata = DocMetadata(subject='01/2004', title='Test Book')
with patch('modules.docs.extract_metadata', return_value=fake_metadata):
await refresh_files()
with get_db_session() as session:
col = session.query(Collection).filter_by(kind='subject', name='01/2004').one_or_none()
assert not col, 'Should not create collection for date-like subject'
@pytest.mark.asyncio
async def test_doc_modeler_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory,
refresh_files):
"""A cover file near the eBook is discovered."""
cover_path = example_epub.with_suffix('.jpg')
cover_path.write_bytes(image_bytes_factory())
await refresh_files()
doc: Doc = test_session.query(Doc).one()
assert doc.cover_path.read_bytes() == cover_path.read_bytes()
@pytest.mark.asyncio
async def test_doc_modeler_extract_cover(test_session, test_directory, example_epub, refresh_files):
"""First image is extracted from the Ebook and used as the cover."""
await refresh_files()
doc: Doc = test_session.query(Doc).one()
# The WROLPi logo is the first image in the example epub.
assert doc.cover_path and doc.cover_path.stat().st_size == 297099
@pytest.mark.asyncio
async def test_doc_modeler_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory,
refresh_files, await_background_tasks):
"""An ebook is re-indexed when moved."""
tag = await tag_factory()
shutil.move(image_file, example_epub.with_suffix('.jpg'))
await refresh_files()
doc: Doc = test_session.query(Doc).one()
assert doc.cover_path
assert_dict_contains(doc.file_group.data, dict(author='roland'))
assert sorted([i['path'] for i in doc.file_group.my_files()]) == [
test_directory / 'ebook example.epub',
test_directory / 'ebook example.jpg',
]
doc.file_group.add_tag(test_session, tag.id)
test_session.commit()
new_directory = test_directory / 'new'
new_directory.mkdir()
file_worker.queue_move(new_directory, [doc.file_group.primary_path])
await await_background_tasks()
test_session.expire_all()
doc: Doc = test_session.query(Doc).one()
assert doc.file_group.primary_path == test_directory / 'new/ebook example.epub'
assert doc.cover_path
assert_dict_contains(doc.file_group.data, dict(author='roland'))
assert sorted([i['path'] for i in doc.file_group.my_files()]) == [
test_directory / 'new/ebook example.epub',
test_directory / 'new/ebook example.jpg',
]
@pytest.mark.asyncio
async def test_doc_modeler_pdf_max_size(test_session, example_pdf):
"""The contents of a large PDF are not indexed."""
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
files_lib._upsert_files([example_pdf], now())
await doc_modeler()
file_group = test_session.query(FileGroup).one()
assert file_group.d_text is None
@pytest.mark.asyncio
async def test_doc_modeler_pdf_poster(test_session, example_pdf):
"""PDF modeler finds an image with the same stem and uses it as the poster."""
poster_path = example_pdf.with_suffix('.jpg')
Image.new('RGB', (25, 25), color='grey').save(poster_path)
files_lib._upsert_files([example_pdf, poster_path], now())
await doc_modeler()
file_group = test_session.query(FileGroup).one()
assert file_group.indexed is True
assert file_group.data is not None
assert file_group.data.get('cover_path') == poster_path.name

View file

@ -0,0 +1,142 @@
import pytest
from modules.docs.extractors import extract_epub, extract_pdf, extract_metadata, extract_docx, extract_odt, \
extract_doc, extract_cbz, DocMetadata
from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_extract_epub(test_session, test_directory, example_epub):
"""Metadata can be extracted from an EPUB file."""
metadata = extract_epub(example_epub)
assert metadata.title == 'WROLPi Test Book'
assert metadata.author == 'roland'
assert metadata.text # Has text content.
assert metadata.cover_bytes # Has a cover image.
@pytest.mark.asyncio
async def test_extract_pdf(test_session, test_directory, example_pdf):
"""Metadata can be extracted from a PDF file."""
metadata = extract_pdf(example_pdf)
# PDF may or may not have title metadata; at minimum it should return something.
assert metadata is not None
assert isinstance(metadata, DocMetadata)
@pytest.mark.asyncio
async def test_extract_pdf_metadata(test_session, test_directory, example_pdf):
"""PDF extractor extracts page count, date, cover, and text."""
metadata = extract_pdf(example_pdf)
assert metadata.title == 'WROLPi Test PDF'
assert metadata.author == 'roland'
assert metadata.page_count == 3
assert metadata.published_date is not None
assert metadata.cover_bytes
assert metadata.text
@pytest.mark.asyncio
async def test_extract_docx(test_session, test_directory, example_docx):
"""DOCX extractor extracts text content."""
metadata = extract_docx(example_docx)
assert isinstance(metadata, DocMetadata)
assert metadata.title == 'Word Document'
assert metadata.text
assert 'Example Word Document' in metadata.text
@pytest.mark.asyncio
async def test_extract_odt(test_session, test_directory, example_odt):
"""ODT extractor extracts title and text."""
metadata = extract_odt(example_odt)
assert metadata.title is not None
assert metadata.text
assert 'Test Header' in metadata.text
@pytest.mark.asyncio
async def test_extract_doc(test_session, test_directory, example_doc):
"""DOC extractor extracts title from filename and text if catdoc/textutil available."""
metadata = extract_doc(example_doc)
assert metadata.title # At minimum, title from filename.
@pytest.mark.asyncio
async def test_extract_cbz(test_session, test_directory, example_cbz):
"""CBZ extractor extracts cover image."""
metadata = extract_cbz(example_cbz)
assert metadata.cover_bytes is not None
assert len(metadata.cover_bytes) > 0
@pytest.mark.asyncio
async def test_extract_cbz_comicinfo(test_session, test_directory, example_cbz_metadata):
"""CBZ extractor reads ComicInfo.xml metadata."""
metadata = extract_cbz(example_cbz_metadata)
assert metadata.title == 'Bobby Make-Believe #1'
assert metadata.author == 'Frank King'
assert metadata.publisher == 'Chicago Tribune'
assert metadata.description == 'Bobby Make-Believe is a comic strip by Frank King.'
assert metadata.language == 'en'
assert metadata.subject == 'Comic Strip'
assert metadata.page_count == 4
assert metadata.published_date.year == 1915
assert metadata.published_date.month == 6
assert metadata.published_date.day == 15
assert metadata.cover_bytes is not None
@pytest.mark.asyncio
async def test_extract_cbt(test_session, test_directory, example_cbt):
"""CBT (TAR) extractor extracts cover image."""
metadata = extract_cbz(example_cbt)
assert metadata.cover_bytes is not None
assert len(metadata.cover_bytes) > 0
@pytest.mark.asyncio
async def test_extract_cbr(test_session, test_directory, example_cbr):
"""CBR (RAR) extractor extracts cover image."""
metadata = extract_cbz(example_cbr)
assert metadata.cover_bytes is not None
assert len(metadata.cover_bytes) > 0
@pytest.mark.asyncio
async def test_extract_cb7(test_session, test_directory, example_cb7):
"""CB7 (7z) extractor extracts cover image."""
metadata = extract_cbz(example_cb7)
assert metadata.cover_bytes is not None
assert len(metadata.cover_bytes) > 0
@pytest.mark.asyncio
async def test_extract_metadata_cbt(test_session, test_directory, example_cbt):
"""extract_metadata dispatches CBT files to the comic extractor."""
fg = FileGroup.from_paths(test_session, example_cbt)
test_session.flush()
metadata = extract_metadata(fg)
assert metadata.cover_bytes is not None
assert len(metadata.cover_bytes) > 0
@pytest.mark.asyncio
async def test_extract_metadata_cb7(test_session, test_directory, example_cb7):
"""extract_metadata dispatches CB7 files to the comic extractor."""
fg = FileGroup.from_paths(test_session, example_cb7)
test_session.flush()
metadata = extract_metadata(fg)
assert metadata.cover_bytes is not None
assert len(metadata.cover_bytes) > 0
@pytest.mark.asyncio
async def test_extract_metadata_dispatches(test_session, test_directory, example_epub):
"""extract_metadata dispatches to the correct extractor based on mimetype."""
fg = FileGroup.from_paths(test_session, example_epub)
test_session.flush()
metadata = extract_metadata(fg)
assert metadata.title == 'WROLPi Test Book'
assert metadata.author == 'roland'

View file

@ -0,0 +1,250 @@
import pytest
from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \
get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \
search_authors_by_name, search_subjects_by_name
def test_is_valid_author():
"""Author validation rejects junk values."""
assert is_valid_author('John Smith')
assert is_valid_author('J.K. Rowling')
assert is_valid_author('Dr. Seuss')
# Reject junk.
assert not is_valid_author('')
assert not is_valid_author(None)
assert not is_valid_author('Administrator')
assert not is_valid_author('administrator')
assert not is_valid_author('Preferred Customer')
assert not is_valid_author('Unknown')
assert not is_valid_author('www.example.com')
assert not is_valid_author('https://example.com')
assert not is_valid_author('A') # Too short.
assert not is_valid_author('x' * 101) # Too long.
# Reject year ranges and pure numbers.
assert not is_valid_author('1483-1546')
assert not is_valid_author('1564-1616')
assert not is_valid_author('-1938')
assert not is_valid_author('1851-1915')
assert not is_valid_author('1114690')
assert not is_valid_author('1588-1679')
# Reject strings with no letters.
assert not is_valid_author('12345')
assert not is_valid_author('---')
def test_split_authors():
"""Author strings with multiple authors are split correctly."""
assert split_authors('John Smith') == ['John Smith']
assert split_authors('John Smith; Jane Doe') == ['John Smith', 'Jane Doe']
assert split_authors('John Smith & Jane Doe') == ['John Smith', 'Jane Doe']
assert split_authors('') == []
assert split_authors(None) == []
def test_normalize_author():
"""Author names are cleaned up before validation."""
# Strip surrounding quotes.
assert normalize_author('"Tony R. Kuphaldt"') == 'Tony R. Kuphaldt'
assert normalize_author('"Mrs. D.A. Lincoln"') == 'Mrs. D.A. Lincoln'
# Remove bracket suffixes.
assert normalize_author('"Mrs. A. J. Barnes." [from old catalog]') == 'Mrs. A. J. Barnes'
assert normalize_author('Catherine] [from old catalog]') == 'Catherine'
assert normalize_author('1851-1915. [from old catalog]') == '1851-1915'
assert normalize_author('"Mrs. John Sterling," [from old catalog] ed') == 'Mrs. John Sterling'
# Remove trailing birth-death years.
assert normalize_author('Caroline Althea (Stickney) 1843-1920') == 'Caroline Althea (Stickney)'
assert normalize_author('1851-1917') == '1851-1917' # All numeric, nothing to strip — normalization passes through.
# Strip trailing period (but not initials).
assert normalize_author('Carol Ann.') == 'Carol Ann'
assert normalize_author('J.K.') == 'J.K.' # Initials stay.
assert normalize_author('Dr. Seuss') == 'Dr. Seuss' # Mid-string period stays.
# Surrounding parentheses.
assert normalize_author('(Thomas Joseph Workman)') == 'Thomas Joseph Workman'
assert normalize_author('(Roland)') == 'Roland'
# Title-case normalization.
assert normalize_author('john smith') == 'John Smith'
assert normalize_author('JOHN SMITH') == 'John Smith'
assert normalize_author('John Smith') == 'John Smith'
assert normalize_author('j.k. rowling') == 'J.K. Rowling'
# Empty/None.
assert normalize_author('') is None
assert normalize_author(None) is None
assert normalize_author(' ') is None
def test_is_valid_subject():
"""Subject validation rejects junk values."""
# Valid subjects.
assert is_valid_subject('Farming')
assert is_valid_subject('Computer Science')
assert is_valid_subject('First Aid')
assert is_valid_subject('Science, Technology and Society')
# Reject empty/short/long.
assert not is_valid_subject('')
assert not is_valid_subject(None)
assert not is_valid_subject('A')
assert not is_valid_subject('x' * 151)
# Reject pure numeric/codes.
assert not is_valid_subject('00110001')
assert not is_valid_subject('062447')
assert not is_valid_subject('105642-01')
assert not is_valid_subject('106763-01')
# Reject date-like strings.
assert not is_valid_subject('01/2004')
assert not is_valid_subject('04/2005')
assert not is_valid_subject('14 June 2006')
# Reject HTML entities.
assert not is_valid_subject('something &bull; else')
# Reject URLs.
assert not is_valid_subject('https://example.com')
assert not is_valid_subject('www.example.com')
# Reject junk names.
assert not is_valid_subject('unknown')
assert not is_valid_subject('n/a')
def test_normalize_subject():
"""Subject strings are cleaned up before validation."""
# Strip whitespace.
assert normalize_subject(' Farming ') == 'Farming'
# Decode HTML entities.
assert normalize_subject('First &amp; Second') == 'First & Second'
# Truncate long strings.
long_subject = 'x' * 200
assert len(normalize_subject(long_subject)) == 150
# Title-case normalization.
assert normalize_subject('science fiction') == 'Science Fiction'
assert normalize_subject('SCIENCE FICTION') == 'Science Fiction'
assert normalize_subject('fairy tales') == 'Fairy Tales'
assert normalize_subject('Fairy Tales') == 'Fairy Tales'
# Empty/None.
assert normalize_subject('') is None
assert normalize_subject(None) is None
assert normalize_subject(' ') is None
def test_split_subjects():
"""Subject strings with multiple subjects are split correctly."""
# Split on bare commas (no trailing space).
assert split_subjects('First Aid,GENERAL PREP.,HEALTH') == ['First Aid', 'GENERAL PREP.', 'HEALTH']
# Split on semicolons.
assert split_subjects('Science; Technology') == ['Science', 'Technology']
# Preserve commas followed by space (legitimate compound subjects).
assert split_subjects('Science, Technology and Society') == ['Science, Technology and Society']
# Empty/None.
assert split_subjects('') == []
assert split_subjects(None) == []
def test_get_or_create_author_collection(test_session):
"""Author collections are created and reused."""
col1 = get_or_create_author_collection(test_session, 'John Smith')
assert col1 is not None
assert col1.name == 'John Smith'
assert col1.kind == 'author'
# Same author returns same collection.
col2 = get_or_create_author_collection(test_session, 'John Smith')
assert col1.id == col2.id
# Different author creates new collection.
col3 = get_or_create_author_collection(test_session, 'Jane Doe')
assert col3.id != col1.id
# Empty author returns None.
assert get_or_create_author_collection(test_session, '') is None
assert get_or_create_author_collection(test_session, None) is None
def test_get_or_create_subject_collection(test_session):
"""Subject collections are created and reused."""
col1 = get_or_create_subject_collection(test_session, 'Farming')
assert col1 is not None
assert col1.name == 'Farming'
assert col1.kind == 'subject'
col2 = get_or_create_subject_collection(test_session, 'Farming')
assert col1.id == col2.id
assert get_or_create_subject_collection(test_session, '') is None
def test_get_or_create_subject_collection_case_insensitive(test_session):
"""Subject collections with different casing reuse the same collection."""
col1 = get_or_create_subject_collection(test_session, 'Science Fiction')
col2 = get_or_create_subject_collection(test_session, 'science fiction')
col3 = get_or_create_subject_collection(test_session, 'SCIENCE FICTION')
assert col1.id == col2.id == col3.id
def test_get_or_create_author_collection_case_insensitive(test_session):
"""Author collections with different casing reuse the same collection."""
col1 = get_or_create_author_collection(test_session, 'John Smith')
col2 = get_or_create_author_collection(test_session, 'john smith')
col3 = get_or_create_author_collection(test_session, 'JOHN SMITH')
assert col1.id == col2.id == col3.id
@pytest.mark.asyncio
async def test_search_authors_by_name(test_session):
"""Search for author collections by partial name."""
get_or_create_author_collection(test_session, 'John Smith')
get_or_create_author_collection(test_session, 'Jane Doe')
get_or_create_author_collection(test_session, 'Johnny Appleseed')
# Partial match.
results = await search_authors_by_name(test_session, 'john')
assert len(results) == 2
names = [r['name'] for r in results]
assert 'John Smith' in names
assert 'Johnny Appleseed' in names
# No match.
results = await search_authors_by_name(test_session, 'nobody')
assert len(results) == 0
# Limit.
results = await search_authors_by_name(test_session, 'j', limit=1)
assert len(results) == 1
@pytest.mark.asyncio
async def test_search_subjects_by_name(test_session):
"""Search for subject collections by partial name."""
get_or_create_subject_collection(test_session, 'Science Fiction')
get_or_create_subject_collection(test_session, 'Computer Science')
get_or_create_subject_collection(test_session, 'History')
# Partial match.
results = await search_subjects_by_name(test_session, 'science')
assert len(results) == 2
names = [r['name'] for r in results]
assert 'Science Fiction' in names
assert 'Computer Science' in names
# No match.
results = await search_subjects_by_name(test_session, 'nothing')
assert len(results) == 0

View file

@ -0,0 +1,56 @@
from modules.docs.models import Doc, DOC_MIMETYPES, mimetype_is_doc
def test_doc_mimetypes():
"""All expected mimetypes are included."""
assert 'application/epub+zip' in DOC_MIMETYPES
assert 'application/x-mobipocket-ebook' in DOC_MIMETYPES
assert 'application/pdf' in DOC_MIMETYPES
assert 'application/msword' in DOC_MIMETYPES
assert 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' in DOC_MIMETYPES
assert 'application/vnd.oasis.opendocument.text' in DOC_MIMETYPES
assert 'application/x-cbz' in DOC_MIMETYPES
assert 'application/x-cbr' in DOC_MIMETYPES
def test_mimetype_is_doc():
"""mimetype_is_doc correctly identifies doc mimetypes."""
assert mimetype_is_doc('application/pdf')
assert mimetype_is_doc('application/epub+zip')
assert mimetype_is_doc('application/x-mobipocket-ebook')
assert mimetype_is_doc('application/msword')
assert not mimetype_is_doc('video/mp4')
assert not mimetype_is_doc('text/plain')
assert not mimetype_is_doc('image/jpeg')
def test_doc_model(test_session, doc_factory):
"""Doc model can be created and queried."""
doc = doc_factory(size=1024, publisher='Test Publisher', language='en', subject='Testing')
assert doc.id
assert doc.size == 1024
assert doc.publisher == 'Test Publisher'
assert doc.language == 'en'
assert doc.subject == 'Testing'
assert doc.file_group is not None
queried = test_session.query(Doc).filter_by(id=doc.id).one()
assert queried.publisher == 'Test Publisher'
def test_doc_can_model(test_session, test_directory):
"""Doc.can_model identifies correct FileGroups."""
from wrolpi.files.models import FileGroup
pdf_path = test_directory / 'test.pdf'
pdf_path.write_bytes(b'%PDF-1.4 test')
fg = FileGroup.from_paths(test_session, pdf_path)
test_session.flush()
assert Doc.can_model(fg)
# Video should not be modeled as a doc.
video_path = test_directory / 'test.mp4'
video_path.write_bytes(b'\x00\x00\x00\x1cftypisom')
fg2 = FileGroup.from_paths(test_session, video_path)
test_session.flush()
assert not Doc.can_model(fg2)

View file

@ -583,9 +583,10 @@ class Channel(ModelHelper, Base):
collection = relationship('Collection', foreign_keys=[collection_id])
@property
def downloads(self) -> InstrumentedList:
"""Get downloads from associated Collection."""
return self.collection.downloads if self.collection else []
def downloads(self) -> list:
"""Get recurring downloads from associated Collection. Once-downloads are excluded because they are not
Channel downloads."""
return [d for d in self.collection.downloads if d.frequency] if self.collection else []
def __repr__(self):
return f'<Channel id={self.id} name={repr(self.name)} url={self.url} directory={self.directory}>'

View file

@ -398,6 +398,39 @@ async def test_import_channel_cleans_up_preexisting_orphaned_collections(await_s
assert test_session.query(Collection).filter_by(id=orphan_id).count() == 0
@pytest.mark.asyncio
async def test_once_download_excluded_from_channel_config(await_switches, test_session, channel_factory,
test_channels_config):
"""A once-download (no frequency) associated with a Channel's Collection should not be saved to the channels
config. This prevents errors during import when a once-download has no frequency."""
channel = channel_factory(download_frequency=DownloadFrequency.weekly)
# Create a once-download associated with the channel's collection (simulates a video download that got
# associated with the channel via apply_yt_channel).
once_download = Download(url='https://www.youtube.com/watch?v=once_video', downloader='video',
collection_id=channel.collection_id)
test_session.add(once_download)
test_session.commit()
# The once-download is in the Collection, but Channel.downloads filters it out.
assert len(channel.collection.downloads) == 2
assert len(channel.downloads) == 1
save_channels_config()
await await_switches()
# The once-download should not appear in the config.
config_text = test_channels_config.read_text()
assert 'once_video' not in config_text
# The recurring download should still be there.
assert channel.url in config_text
# Importing should succeed without the "Refusing to create Download" error.
test_session.delete(once_download)
test_session.commit()
import_channels_config()
assert len(channel.downloads) == 1
@pytest.mark.asyncio
async def test_import_channel_download_comments(await_switches, test_session, channel_factory,
test_channels_config):

View file

@ -93,6 +93,7 @@ sysstat
tar
tcpdump
tmux
unrar-free
unzip
vim
vlc

View file

@ -52,7 +52,6 @@ chown -R _renderd:_renderd /var/cache/renderd/tiles
/opt/wrolpi/scripts/install_help_service.sh
# Create the media directory for the wrolpi user.
echo '/dev/sda1 /media/wrolpi auto defaults,nofail 0 0' | tee -a /etc/fstab
mkdir -p /media/wrolpi
chown -R wrolpi:wrolpi /media/wrolpi /home/wrolpi /opt/wrolpi*

View file

@ -19,10 +19,12 @@ PyMuPDF>=1.27.0
pytest-asyncio==0.24.0
pytest-xdist==3.5.0
pytest==8.3.3
py7zr>=1.1.0
python-docx==1.1.2
python-dotenv==1.0.1
python-magic~=0.4.27
pytz==2024.2
rarfile>=4.2
sanic-ext==25.12.0
sanic-testing==24.6.0
sanic==25.12.0

View file

@ -0,0 +1,504 @@
GNU LESSER GENERAL PUBLIC LICENSE
Version 2.1, February 1999
Copyright (C) 1991, 1999 Free Software Foundation, Inc.
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
[This is the first released version of the Lesser GPL. It also counts
as the successor of the GNU Library Public License, version 2, hence
the version number 2.1.]
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
Licenses are intended to guarantee your freedom to share and change
free software--to make sure the software is free for all its users.
This license, the Lesser General Public License, applies to some
specially designated software packages--typically libraries--of the
Free Software Foundation and other authors who decide to use it. You
can use it too, but we suggest you first think carefully about whether
this license or the ordinary General Public License is the better
strategy to use in any particular case, based on the explanations below.
When we speak of free software, we are referring to freedom of use,
not price. Our General Public Licenses are designed to make sure that
you have the freedom to distribute copies of free software (and charge
for this service if you wish); that you receive source code or can get
it if you want it; that you can change the software and use pieces of
it in new free programs; and that you are informed that you can do
these things.
To protect your rights, we need to make restrictions that forbid
distributors to deny you these rights or to ask you to surrender these
rights. These restrictions translate to certain responsibilities for
you if you distribute copies of the library or if you modify it.
For example, if you distribute copies of the library, whether gratis
or for a fee, you must give the recipients all the rights that we gave
you. You must make sure that they, too, receive or can get the source
code. If you link other code with the library, you must provide
complete object files to the recipients, so that they can relink them
with the library after making changes to the library and recompiling
it. And you must show them these terms so they know their rights.
We protect your rights with a two-step method: (1) we copyright the
library, and (2) we offer you this license, which gives you legal
permission to copy, distribute and/or modify the library.
To protect each distributor, we want to make it very clear that
there is no warranty for the free library. Also, if the library is
modified by someone else and passed on, the recipients should know
that what they have is not the original version, so that the original
author's reputation will not be affected by problems that might be
introduced by others.
Finally, software patents pose a constant threat to the existence of
any free program. We wish to make sure that a company cannot
effectively restrict the users of a free program by obtaining a
restrictive license from a patent holder. Therefore, we insist that
any patent license obtained for a version of the library must be
consistent with the full freedom of use specified in this license.
Most GNU software, including some libraries, is covered by the
ordinary GNU General Public License. This license, the GNU Lesser
General Public License, applies to certain designated libraries, and
is quite different from the ordinary General Public License. We use
this license for certain libraries in order to permit linking those
libraries into non-free programs.
When a program is linked with a library, whether statically or using
a shared library, the combination of the two is legally speaking a
combined work, a derivative of the original library. The ordinary
General Public License therefore permits such linking only if the
entire combination fits its criteria of freedom. The Lesser General
Public License permits more lax criteria for linking other code with
the library.
We call this license the "Lesser" General Public License because it
does Less to protect the user's freedom than the ordinary General
Public License. It also provides other free software developers Less
of an advantage over competing non-free programs. These disadvantages
are the reason we use the ordinary General Public License for many
libraries. However, the Lesser license provides advantages in certain
special circumstances.
For example, on rare occasions, there may be a special need to
encourage the widest possible use of a certain library, so that it becomes
a de-facto standard. To achieve this, non-free programs must be
allowed to use the library. A more frequent case is that a free
library does the same job as widely used non-free libraries. In this
case, there is little to gain by limiting the free library to free
software only, so we use the Lesser General Public License.
In other cases, permission to use a particular library in non-free
programs enables a greater number of people to use a large body of
free software. For example, permission to use the GNU C Library in
non-free programs enables many more people to use the whole GNU
operating system, as well as its variant, the GNU/Linux operating
system.
Although the Lesser General Public License is Less protective of the
users' freedom, it does ensure that the user of a program that is
linked with the Library has the freedom and the wherewithal to run
that program using a modified version of the Library.
The precise terms and conditions for copying, distribution and
modification follow. Pay close attention to the difference between a
"work based on the library" and a "work that uses the library". The
former contains code derived from the library, whereas the latter must
be combined with the library in order to run.
GNU LESSER GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License Agreement applies to any software library or other
program which contains a notice placed by the copyright holder or
other authorized party saying it may be distributed under the terms of
this Lesser General Public License (also called "this License").
Each licensee is addressed as "you".
A "library" means a collection of software functions and/or data
prepared so as to be conveniently linked with application programs
(which use some of those functions and data) to form executables.
The "Library", below, refers to any such software library or work
which has been distributed under these terms. A "work based on the
Library" means either the Library or any derivative work under
copyright law: that is to say, a work containing the Library or a
portion of it, either verbatim or with modifications and/or translated
straightforwardly into another language. (Hereinafter, translation is
included without limitation in the term "modification".)
"Source code" for a work means the preferred form of the work for
making modifications to it. For a library, complete source code means
all the source code for all modules it contains, plus any associated
interface definition files, plus the scripts used to control compilation
and installation of the library.
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running a program using the Library is not restricted, and output from
such a program is covered only if its contents constitute a work based
on the Library (independent of the use of the Library in a tool for
writing it). Whether that is true depends on what the Library does
and what the program that uses the Library does.
1. You may copy and distribute verbatim copies of the Library's
complete source code as you receive it, in any medium, provided that
you conspicuously and appropriately publish on each copy an
appropriate copyright notice and disclaimer of warranty; keep intact
all the notices that refer to this License and to the absence of any
warranty; and distribute a copy of this License along with the
Library.
You may charge a fee for the physical act of transferring a copy,
and you may at your option offer warranty protection in exchange for a
fee.
2. You may modify your copy or copies of the Library or any portion
of it, thus forming a work based on the Library, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) The modified work must itself be a software library.
b) You must cause the files modified to carry prominent notices
stating that you changed the files and the date of any change.
c) You must cause the whole of the work to be licensed at no
charge to all third parties under the terms of this License.
d) If a facility in the modified Library refers to a function or a
table of data to be supplied by an application program that uses
the facility, other than as an argument passed when the facility
is invoked, then you must make a good faith effort to ensure that,
in the event an application does not supply such function or
table, the facility still operates, and performs whatever part of
its purpose remains meaningful.
(For example, a function in a library to compute square roots has
a purpose that is entirely well-defined independent of the
application. Therefore, Subsection 2d requires that any
application-supplied function or table used by this function must
be optional: if the application does not supply it, the square
root function must still compute square roots.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Library,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Library, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote
it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Library.
In addition, mere aggregation of another work not based on the Library
with the Library (or with a work based on the Library) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may opt to apply the terms of the ordinary GNU General Public
License instead of this License to a given copy of the Library. To do
this, you must alter all the notices that refer to this License, so
that they refer to the ordinary GNU General Public License, version 2,
instead of to this License. (If a newer version than version 2 of the
ordinary GNU General Public License has appeared, then you can specify
that version instead if you wish.) Do not make any other change in
these notices.
Once this change is made in a given copy, it is irreversible for
that copy, so the ordinary GNU General Public License applies to all
subsequent copies and derivative works made from that copy.
This option is useful when you wish to copy part of the code of
the Library into a program that is not a library.
4. You may copy and distribute the Library (or a portion or
derivative of it, under Section 2) in object code or executable form
under the terms of Sections 1 and 2 above provided that you accompany
it with the complete corresponding machine-readable source code, which
must be distributed under the terms of Sections 1 and 2 above on a
medium customarily used for software interchange.
If distribution of object code is made by offering access to copy
from a designated place, then offering equivalent access to copy the
source code from the same place satisfies the requirement to
distribute the source code, even though third parties are not
compelled to copy the source along with the object code.
5. A program that contains no derivative of any portion of the
Library, but is designed to work with the Library by being compiled or
linked with it, is called a "work that uses the Library". Such a
work, in isolation, is not a derivative work of the Library, and
therefore falls outside the scope of this License.
However, linking a "work that uses the Library" with the Library
creates an executable that is a derivative of the Library (because it
contains portions of the Library), rather than a "work that uses the
library". The executable is therefore covered by this License.
Section 6 states terms for distribution of such executables.
When a "work that uses the Library" uses material from a header file
that is part of the Library, the object code for the work may be a
derivative work of the Library even though the source code is not.
Whether this is true is especially significant if the work can be
linked without the Library, or if the work is itself a library. The
threshold for this to be true is not precisely defined by law.
If such an object file uses only numerical parameters, data
structure layouts and accessors, and small macros and small inline
functions (ten lines or less in length), then the use of the object
file is unrestricted, regardless of whether it is legally a derivative
work. (Executables containing this object code plus portions of the
Library will still fall under Section 6.)
Otherwise, if the work is a derivative of the Library, you may
distribute the object code for the work under the terms of Section 6.
Any executables containing that work also fall under Section 6,
whether or not they are linked directly with the Library itself.
6. As an exception to the Sections above, you may also combine or
link a "work that uses the Library" with the Library to produce a
work containing portions of the Library, and distribute that work
under terms of your choice, provided that the terms permit
modification of the work for the customer's own use and reverse
engineering for debugging such modifications.
You must give prominent notice with each copy of the work that the
Library is used in it and that the Library and its use are covered by
this License. You must supply a copy of this License. If the work
during execution displays copyright notices, you must include the
copyright notice for the Library among them, as well as a reference
directing the user to the copy of this License. Also, you must do one
of these things:
a) Accompany the work with the complete corresponding
machine-readable source code for the Library including whatever
changes were used in the work (which must be distributed under
Sections 1 and 2 above); and, if the work is an executable linked
with the Library, with the complete machine-readable "work that
uses the Library", as object code and/or source code, so that the
user can modify the Library and then relink to produce a modified
executable containing the modified Library. (It is understood
that the user who changes the contents of definitions files in the
Library will not necessarily be able to recompile the application
to use the modified definitions.)
b) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (1) uses at run time a
copy of the library already present on the user's computer system,
rather than copying library functions into the executable, and (2)
will operate properly with a modified version of the library, if
the user installs one, as long as the modified version is
interface-compatible with the version that the work was made with.
c) Accompany the work with a written offer, valid for at
least three years, to give the same user the materials
specified in Subsection 6a, above, for a charge no more
than the cost of performing this distribution.
d) If distribution of the work is made by offering access to copy
from a designated place, offer equivalent access to copy the above
specified materials from the same place.
e) Verify that the user has already received a copy of these
materials or that you have already sent this user a copy.
For an executable, the required form of the "work that uses the
Library" must include any data and utility programs needed for
reproducing the executable from it. However, as a special exception,
the materials to be distributed need not include anything that is
normally distributed (in either source or binary form) with the major
components (compiler, kernel, and so on) of the operating system on
which the executable runs, unless that component itself accompanies
the executable.
It may happen that this requirement contradicts the license
restrictions of other proprietary libraries that do not normally
accompany the operating system. Such a contradiction means you cannot
use both them and the Library together in an executable that you
distribute.
7. You may place library facilities that are a work based on the
Library side-by-side in a single library together with other library
facilities not covered by this License, and distribute such a combined
library, provided that the separate distribution of the work based on
the Library and of the other library facilities is otherwise
permitted, and provided that you do these two things:
a) Accompany the combined library with a copy of the same work
based on the Library, uncombined with any other library
facilities. This must be distributed under the terms of the
Sections above.
b) Give prominent notice with the combined library of the fact
that part of it is a work based on the Library, and explaining
where to find the accompanying uncombined form of the same work.
8. You may not copy, modify, sublicense, link with, or distribute
the Library except as expressly provided under this License. Any
attempt otherwise to copy, modify, sublicense, link with, or
distribute the Library is void, and will automatically terminate your
rights under this License. However, parties who have received copies,
or rights, from you under this License will not have their licenses
terminated so long as such parties remain in full compliance.
9. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Library or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Library (or any work based on the
Library), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Library or works based on it.
10. Each time you redistribute the Library (or any work based on the
Library), the recipient automatically receives a license from the
original licensor to copy, distribute, link with or modify the Library
subject to these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties with
this License.
11. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Library at all. For example, if a patent
license would not permit royalty-free redistribution of the Library by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Library.
If any portion of this section is held invalid or unenforceable under any
particular circumstance, the balance of the section is intended to apply,
and the section as a whole is intended to apply in other circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
12. If the distribution and/or use of the Library is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Library under this License may add
an explicit geographical distribution limitation excluding those countries,
so that distribution is permitted only in or among countries not thus
excluded. In such case, this License incorporates the limitation as if
written in the body of this License.
13. The Free Software Foundation may publish revised and/or new
versions of the Lesser General Public License from time to time.
Such new versions will be similar in spirit to the present version,
but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Library
specifies a version number of this License which applies to it and
"any later version", you have the option of following the terms and
conditions either of that version or of any later version published by
the Free Software Foundation. If the Library does not specify a
license version number, you may choose any version ever published by
the Free Software Foundation.
14. If you wish to incorporate parts of the Library into other free
programs whose distribution conditions are incompatible with these,
write to the author to ask for permission. For software which is
copyrighted by the Free Software Foundation, write to the Free
Software Foundation; we sometimes make exceptions for this. Our
decision will be guided by the two goals of preserving the free status
of all derivatives of our free software and of promoting the sharing
and reuse of software generally.
NO WARRANTY
15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Libraries
If you develop a new library, and you want it to be of the greatest
possible use to the public, we recommend making it free software that
everyone can redistribute and change. You can do so by permitting
redistribution under these terms (or, alternatively, under the terms of the
ordinary General Public License).
To apply these terms, attach the following notices to the library. It is
safest to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least the
"copyright" line and a pointer to where the full notice is found.
<one line to give the library's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
USA
Also add information on how to contact you by electronic and paper mail.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the library, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the
library `Frob' (a library for tweaking knobs) written by James Random
Hacker.
<signature of Ty Coon>, 1 April 1990
Ty Coon, President of Vice
That's all there is to it!

View file

@ -0,0 +1,81 @@
# sample_reading_media
Test sample ebooks, etc. Formats:
* test_book.md
* test_book_md.md
* test_book_txt.txt
* test_book_txt_crlf_win.txt
* test_book_txt_lf_unix.txt
* test_book_html.html
* test_book_odt.odt
* test_book_docx.docx
* test_book_pdf.pdf
* test_book_fb2.fb2
* test_book_epub.epub
* test_book_md_zip.zip
* test_book_txt_zip.zip
* test_book_html_zip.zip
* test_book_fb2_zip.zip
* test_book_rtf.rtf
* test_book_rtf_zip.zip
All the above sample books are generated from [test_book.md](./test_book.md).
* source_test_book_fb2.fb2
* source_test_book_fb2_zip.fbz
* source_test_book_fb2_dot_zip.fb2.zip
* test_book_epub_more_detail.epub
* test_book_pdf_more_detail.pdf
LGPL license, so feel free to use in/with your projects. If you modify them, share your changes.
Test sample comics, Formats:
* bobby_make_believe_sample.cb7
* bobby_make_believe_sample.cbt
* bobby_make_believe_sample.cbz
* bobby_make_believe_sample.cbr
* bobby_make_believe_sample_dir.cb7
* bobby_make_believe_sample_dir.cbt
* bobby_make_believe_sample_dir.cbz
For Comic book also sample, see https://www.contrapositivediary.com/?p=1197
(e.g. http://www.copperwood.com/pub/Elf%20Receiver%20Radio-Craft%20August%201936.cbz)
Images in images/bobby_make_believe/ are in the Public Domain and are the first 4 four pages of "Bobby Make-Believe (1915)" from https://comicbookplus.com/?dlid=26481
## Build setup
# Assuming Debian based
sudo apt install pandoc zip
sudo apt install wget
sudo apt install p7zip-full
sudo apt install rar
Issue build:
./build.sh
## TODO
* add images (e.g. to html, embedded and linked, epub, new md file with images etc.)
* fb2 with title page/images
* hand crafted html (e.g. include metadata)
* mobi
* prc/pdb?
* azw
* azw3
* add comics
* CBR
* epub with images only
* metadata for formats that support it
* mobi
* azw/azw3
* Comic metadata; zip comment or a ComicInfo.xml - see
* https://github.com/dickloraine/EmbedComicMetadata
* https://github.com/comictagger/comictagger
* https://code.google.com/archive/p/comicbookinfo/ (wiki)
* https://wiki.mobileread.com/wiki/CBR_and_CBZ#Metadata

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -127,6 +127,7 @@ def get_installed_browsers() -> list:
return browsers
# Videos
YT_DLP_BIN = which(
'/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location

View file

@ -4,7 +4,7 @@ Collection Library Functions
Provides generic operations for collections of any kind (domains, channels, etc.).
These functions are used by both the unified collection API and legacy endpoints.
"""
import asyncio
import pathlib
from typing import List, Optional, Dict
from sqlalchemy import func
@ -13,7 +13,6 @@ from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
from wrolpi.db import get_db_session
import pathlib
from wrolpi.errors import FileWorkerConflict
from wrolpi.errors import ValidationError
from wrolpi.events import Events
@ -68,6 +67,7 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
# Separate collections by kind for batch processing
domain_ids = [c.id for c in collections if c.kind == 'domain']
channel_ids = [c.id for c in collections if c.kind == 'channel']
generic_ids = [c.id for c in collections if c.kind not in ('domain', 'channel')]
# Batch query: Get archive stats for all domain collections at once
domain_stats_map = {}
@ -102,6 +102,19 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
channel_stats_map = {s.collection_id: s for s in channel_stats}
# Batch query: Get item counts for generic collections (author, subject, etc.)
generic_stats_map = {}
if generic_ids:
from .models import CollectionItem
generic_stats = session.query(
CollectionItem.collection_id,
func.count(CollectionItem.id).label('item_count'),
).filter(
CollectionItem.collection_id.in_(generic_ids)
).group_by(CollectionItem.collection_id).all()
generic_stats_map = {s.collection_id: s.item_count for s in generic_stats}
# Convert to JSON and merge batch statistics
result = []
for collection in collections:
@ -133,6 +146,10 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
data['total_size'] = 0
data['channel_id'] = None
else:
# Generic collections (author, subject, etc.): use actual CollectionItem count.
data['item_count'] = generic_stats_map.get(collection.id, 0)
result.append(data)
return result
@ -198,6 +215,14 @@ def get_collection_with_stats(session: Session, collection_id: int) -> dict:
data['video_count'] = 0
data['total_size'] = 0
else:
# Generic collections (author, subject, etc.): count actual CollectionItem rows.
from .models import CollectionItem
item_count = session.query(func.count(CollectionItem.id)).filter(
CollectionItem.collection_id == collection_id
).scalar()
data['item_count'] = item_count or 0
return data

View file

@ -1,7 +1,8 @@
import pathlib
from typing import Optional, List
from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func, BigInteger, or_
from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func, BigInteger, \
or_
from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm.collections import InstrumentedList
@ -11,9 +12,9 @@ from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_re
from wrolpi.downloader import Download, save_downloads_config
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.files.worker import file_worker
from wrolpi.files.models import FileGroup
from wrolpi.files.lib import delete_directory
from wrolpi.files.models import FileGroup
from wrolpi.files.worker import file_worker
from wrolpi.media_path import MediaPathType
from wrolpi.tags import Tag
from wrolpi.tags import save_tags_config
@ -58,7 +59,7 @@ def validate_collection_directory(directory: pathlib.Path) -> pathlib.Path:
class Collection(ModelHelper, Base):
"""
A Collection is a grouping of FileGroups (videos, archives, ebooks, etc).
A Collection is a grouping of FileGroups (videos, archives, docs, etc).
Collections can be:
- Directory-restricted: Only contains files within a specific directory tree
@ -290,8 +291,8 @@ class Collection(ModelHelper, Base):
tag_name = data.get('tag_name')
file_format = data.get('file_format')
kind = (data.get('kind') or 'channel').strip()
# Validate known kinds (forward-compatible: allow future values)
if kind not in {'channel', 'domain'}:
# Validate known kinds using the registry (allows extensible kinds).
if not collection_type_registry.is_registered(kind):
logger.warning(f"Unknown collection kind '{kind}', defaulting to 'channel'")
kind = 'channel'
@ -430,8 +431,8 @@ class Collection(ModelHelper, Base):
file_format = data.get('file_format')
kind = (data.get('kind') or 'channel').strip()
# Validate known kinds
if kind not in {'channel', 'domain'}:
# Validate known kinds using the registry.
if not collection_type_registry.is_registered(kind):
logger.warning(f"Unknown collection kind '{kind}', defaulting to 'channel'")
kind = 'channel'

View file

@ -17,7 +17,6 @@ from sqlalchemy.orm import Session, joinedload
from wrolpi.common import logger, get_media_directory
from wrolpi.db import get_db_session
from wrolpi.files.lib import glob_shared_stem
from .errors import UnknownCollection
from .models import Collection

View file

@ -136,6 +136,34 @@ class TestCollectionsAPI:
assert test_channel_data['total_size'] > 0
@pytest.mark.asyncio
async def test_get_collections_includes_item_count_for_author(
self, async_client, test_session, test_directory
):
"""Test that GET /api/collections?kind=author returns correct item_count from CollectionItem rows."""
from modules.docs.lib import get_or_create_author_collection
from wrolpi.files.models import FileGroup
# Create an author collection with a file group added to it.
author_col = get_or_create_author_collection(test_session, 'John Smith')
path = test_directory / 'test_doc.pdf'
path.write_bytes(b'%PDF-1.4 test')
fg = FileGroup.from_paths(test_session, path)
test_session.flush()
author_col.add_file_group(fg, session=test_session)
test_session.commit()
# Make the request.
request, response = await async_client.get('/api/collections?kind=author')
assert response.status_code == HTTPStatus.OK
data = response.json
author_data = next(c for c in data['collections'] if c['name'] == 'John Smith')
assert author_data['item_count'] == 1, \
f'Author collection should report 1 item, got {author_data["item_count"]}'
class TestCollectionTagInfoAPI:
"""Test the POST /api/collections/<id>/tag_info endpoint."""
@ -415,7 +443,7 @@ class TestCollectionDeletion:
self, async_client, test_session, test_directory, channel_factory, video_factory
):
"""Test that deleting a channel collection with videos orphans the videos."""
from modules.videos.models import Channel, Video
from modules.videos.models import Channel
# Create a channel with videos
channel = channel_factory(name='test', directory=test_directory / 'videos' / 'test')

View file

@ -331,7 +331,6 @@ async def test_tag_domain_comprehensive(
8. Tagging without directory doesn't move files
"""
from modules.archive.lib import save_domains_config, get_domains_config
from wrolpi.downloader import Download
from wrolpi.common import walk
# Create domain collection directory in the usual archive directory

View file

@ -70,7 +70,7 @@ def test_collection_format_destination_domain(test_session, test_directory):
@pytest.mark.asyncio
async def test_collection_format_destination_domain_with_tag_template(async_client, test_session, test_directory,
test_wrolpi_config):
test_wrolpi_config):
"""Collection.format_destination uses tag when template includes %(domain_tag)s."""
# Update config to include tag in template
config = get_wrolpi_config()

View file

@ -18,7 +18,6 @@ from wrolpi.collections.reorganize import (
get_batch_reorganization_status,
ReorganizationPreview,
)
from wrolpi.common import get_media_directory
from wrolpi.db import get_db_session
from wrolpi.files.models import FileGroup
@ -874,7 +873,6 @@ async def test_reorganize_updates_tag_directory_links(async_client, test_directo
async def test_batch_reorganize_preserves_tags(async_client, test_directory, video_factory, tag_factory,
test_tags_config):
"""Batch reorganization should preserve tags across multiple collections."""
from wrolpi.files.worker import file_worker
import asyncio
videos_dir = test_directory / 'videos'
@ -1042,7 +1040,6 @@ async def test_reorganize_can_retry_after_partial_failure(async_client, test_dir
- file_format should NOT have been updated (allowing retry)
- needs_reorganization should still return True
"""
from wrolpi.files.worker import file_worker
import shutil
# Create channel collection
@ -1675,7 +1672,7 @@ def test_reorganize_skips_archives_missing_required_metadata(
@pytest.mark.asyncio
async def test_domain_reorganization_moves_files_from_root_to_year_subdirectory(
async_client, test_directory, archive_factory
async_client, test_directory, archive_factory
):
"""Archives in collection root should be moved to year subdirectories.
@ -1756,7 +1753,7 @@ async def test_domain_reorganization_moves_files_from_root_to_year_subdirectory(
@pytest.mark.asyncio
async def test_channel_reorganization_moves_files_from_root_to_year_subdirectory(
async_client, test_directory, video_factory
async_client, test_directory, video_factory
):
"""Videos in collection root should be moved to year subdirectories.
@ -1843,7 +1840,7 @@ async def test_channel_reorganization_moves_files_from_root_to_year_subdirectory
@pytest.mark.asyncio
async def test_channel_reorganization_preserves_root_level_files(
async_client, test_directory, video_factory
async_client, test_directory, video_factory
):
"""Root-level files (like channel info.json and images) should NOT be moved during reorganization.
@ -2298,28 +2295,28 @@ async def test_conflict_preview_includes_quality_rank(async_client, test_directo
# Filename with all fields: uploader_date_sourceid_title.ext
# Should work with complex formats since filename parsing provides all metadata
(
"Learning Self-Reliance_20170529_p_MzsCFkUPU_How to Build a Solar Generator.mp4",
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s",
True, # Should work - filename has uploader, date, and source_id
"Learning Self-Reliance_20170529_p_MzsCFkUPU_How to Build a Solar Generator.mp4",
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s",
True, # Should work - filename has uploader, date, and source_id
),
# Filename without source_id: uploader_date_title.ext pattern
# Should NOT work with format requiring %(id)s
(
"Commsprepper_20130113_110Ah Car Battery connected to 80W Solar Panel.mp4",
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s",
False, # Should fail - filename doesn't have source_id and format requires it
"Commsprepper_20130113_110Ah Car Battery connected to 80W Solar Panel.mp4",
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s",
False, # Should fail - filename doesn't have source_id and format requires it
),
# Filename without source_id but format doesn't require it
(
"Commsprepper_20130113_110Ah Car Battery.mp4",
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s",
False, # Should still fail - all-or-nothing: no source_id means no fallback
"Commsprepper_20130113_110Ah Car Battery.mp4",
"%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s",
False, # Should still fail - all-or-nothing: no source_id means no fallback
),
# Simple title-only format (always works)
(
"random_video_name.mp4",
"%(title)s.%(ext)s",
True, # Should work - format only requires title
"random_video_name.mp4",
"%(title)s.%(ext)s",
True, # Should work - format only requires title
),
])
def test_video_metadata_fallback_from_filename(
@ -2438,42 +2435,42 @@ def test_video_metadata_fallback_uses_channel_name_for_uploader(test_session, te
# Complete filename with all fields - channel name takes priority over filename
# This ensures reorganized files use the WROLPi channel name, not the YouTube uploader
(
"TestUploader_20170529_p_MzsCFkUPU_My Test Video.mp4",
"FormatFallbackChannel",
"My Test Video",
None, # No DB datetime
None, # No DB source_id
"2017",
"FormatFallbackChannel", # Channel name takes priority
"p_MzsCFkUPU",
["TestUploader"], # Should NOT contain filename uploader
"TestUploader_20170529_p_MzsCFkUPU_My Test Video.mp4",
"FormatFallbackChannel",
"My Test Video",
None, # No DB datetime
None, # No DB source_id
"2017",
"FormatFallbackChannel", # Channel name takes priority
"p_MzsCFkUPU",
["TestUploader"], # Should NOT contain filename uploader
),
# Incomplete filename (no source_id) - uses DB values, not filename
(
"FilenameUploader_20200101_Just A Title.mp4",
"ChannelFromDB",
"Just A Title",
"2024-06-15", # DB datetime
"db_source_id", # DB source_id
"2024",
"ChannelFromDB",
"db_source_id",
["FilenameUploader", "2020"], # Should NOT contain these filename-parsed values
"FilenameUploader_20200101_Just A Title.mp4",
"ChannelFromDB",
"Just A Title",
"2024-06-15", # DB datetime
"db_source_id", # DB source_id
"2024",
"ChannelFromDB",
"db_source_id",
["FilenameUploader", "2020"], # Should NOT contain these filename-parsed values
),
],
)
def test_format_video_filename_fallback_behavior(
test_session,
test_directory,
filename,
channel_name,
title,
db_datetime,
db_source_id,
expected_year,
expected_uploader,
expected_source_id,
should_not_contain,
test_session,
test_directory,
filename,
channel_name,
title,
db_datetime,
db_source_id,
expected_year,
expected_uploader,
expected_source_id,
should_not_contain,
):
"""Test format_video_filename fallback behavior with various scenarios.

View file

@ -580,7 +580,6 @@ class ConfigFile:
suffix = pathlib.Path(self.file_name).suffix
return get_media_directory() / f'config/backup/{stem}-{backup_date}{suffix}'
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
"""Preview what a backup import would do. Subclasses override this."""
raise NotImplementedError(f'{self.__class__.__name__} does not support backup import')

View file

@ -249,7 +249,8 @@ class AutoAwaitTestClient:
@pytest.fixture
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[AutoAwaitTestClient, Any]:
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[
AutoAwaitTestClient, Any]:
"""Get an Async Sanic Test Client with all default routes attached.
Depends on test_session to ensure the database mock is in place for session middleware.
@ -536,6 +537,69 @@ def example_docx(test_directory) -> Generator[Path, Any, None]:
yield destination
@pytest.fixture
def example_odt(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'test_book_odt.odt'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/test_book_odt.odt', destination)
yield destination
@pytest.fixture
def example_cbz(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_sample.cbz'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_sample.cbz', destination)
yield destination
@pytest.fixture
def example_cbz_metadata(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_metadata.cbz'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_metadata.cbz', destination)
yield destination
@pytest.fixture
def example_cbt(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_cbt.cbt'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbt.cbt', destination)
yield destination
@pytest.fixture
def example_cbr(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_cbr.cbr'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbr.cbr', destination)
yield destination
@pytest.fixture
def example_cb7(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_cb7.cb7'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cb7.cb7', destination)
yield destination
@pytest.fixture
def example_cbz_dir(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_dir_cbz.cbz'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbz.cbz', destination)
yield destination
@pytest.fixture
def example_cbt_dir(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_dir_cbt.cbt'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbt.cbt', destination)
yield destination
@pytest.fixture
def example_cb7_dir(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_dir_cb7.cb7'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cb7.cb7', destination)
yield destination
@pytest.fixture
def make_files_structure(test_directory) -> Callable[[Union[List[Union[pathlib.Path, str]], dict]], List[pathlib.Path]]:
"""
@ -1038,5 +1102,3 @@ def refresh_files(async_client):
await await_file_worker()
return _refresh

View file

@ -18,8 +18,8 @@ from http import HTTPStatus
from itertools import filterfalse
from typing import List, Dict, Generator, Iterable, Coroutine, Any
from typing import Tuple, Optional
from zoneinfo import ZoneInfo
from urllib.parse import urlparse
from zoneinfo import ZoneInfo
import feedparser
import pytz
@ -31,7 +31,6 @@ from sqlalchemy.sql import Delete
from wrolpi import flags
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which, run_command, CommandResult
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
@ -1752,7 +1751,8 @@ class DownloadManagerConfig(ConfigFile):
shutil.copy2(backup_file, config_file)
elif mode == 'merge':
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(downloads=[], skip_urls=[], version=0)
current_data = self.read_config_file() if config_file.is_file() else dict(downloads=[], skip_urls=[],
version=0)
current_urls = {dl.get('url') for dl in current_data.get('downloads', []) if dl.get('url')}
merged_downloads = list(current_data.get('downloads', []))

View file

@ -1,4 +1,2 @@
from . import downloader # noqa
from . import ebooks # noqa
from . import pdfs # noqa
from .api import files_bp

View file

@ -11,11 +11,10 @@ from sanic_ext.extensions.openapi import openapi
from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory, logger, \
background_task, walk, timer, TRACE_LEVEL, unique_by_predicate
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict, UnsupportedArchive
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
from . import lib, schema
from .worker import file_worker
from ..api_utils import json_response, api_app
from ..events import Events
from ..schema import JSONErrorResponse
from ..tags import Tag

View file

@ -1,350 +0,0 @@
import asyncio
import dataclasses
import logging
import pathlib
from typing import Callable, List, Optional, Tuple
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, or_, and_, Index
from sqlalchemy.orm import relationship, Session
from wrolpi.common import ModelHelper, Base, register_modeler, get_html_soup
from wrolpi.db import get_db_session
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
try:
import ebooklib
from ebooklib import epub
except ImportError:
ebooklib = None
epub = None
logger = logging.getLogger(__name__)
__all__ = [
'EBOOK_MIMETYPES',
'EBook',
'EBookData',
'EPUB_MIMETYPE',
'MOBI_MIMETYPE',
'ebook_modeler',
'extract_ebook_cover',
'extract_ebook_data',
'mimetype_is_ebook',
'model_ebook',
]
EBOOK_SUFFIXES = ('.epub', '.mobi')
EPUB_MIMETYPE = 'application/epub'
MOBI_MIMETYPE = 'application/x-mobipocket-ebook'
EBOOK_MIMETYPES = (
EPUB_MIMETYPE,
MOBI_MIMETYPE,
)
@dataclasses.dataclass
class EBookData:
cover: bytes = None
author: str = None
text: str = None
title: str = None
ebook_path: pathlib.Path = None
def __bool__(self):
return bool(self.cover) or bool(self.author) or bool(self.text) or bool(self.title)
def __json__(self) -> dict:
d = {}
if self.author:
d['author'] = self.author
if self.title:
d['title'] = self.title
if self.ebook_path:
d['ebook_path'] = self.ebook_path
return d
def extract_text(html: str) -> str:
soup = get_html_soup(html)
text = soup.get_text()
return text
def mimetype_is_ebook(mimetype: str) -> bool:
return any(i for i in EBOOK_MIMETYPES if mimetype.startswith(i))
def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]:
"""Extract data from within an eBook file."""
if ebooklib is None or epub is None:
raise ValueError('ebooklib is not installed')
if not mimetype.startswith(EPUB_MIMETYPE):
return None
data = EBookData(ebook_path=path)
book = epub.read_epub(path, options=dict(ignore_ncx=True))
for key, value in book.metadata.items():
if 'title' in value:
data.title = data.title or value['title'][0][0]
if 'creator' in value:
data.author = data.author or value['creator'][0][0]
data.text = ''
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
data.text += extract_text(doc.content)
return data
def extract_ebook_cover(path: pathlib.Path) -> Optional[bytes]:
"""Attempt to find the Cover within an ePub file. If no cover is found, return the first image."""
if ebooklib is None or epub is None:
raise ValueError('ebooklib is not installed')
book = epub.read_epub(path, options=dict(ignore_ncx=True))
for doc in book.get_items_of_type(ebooklib.ITEM_COVER):
logger.debug(f'Found cover for {path}')
return doc.content
# Use the first image as the cover.
for doc in book.get_items_of_type(ebooklib.ITEM_IMAGE):
logger.debug(f'Using first image as cover for {path}')
return doc.content
def discover_calibre_cover(ebook_path: pathlib.Path):
"""Calibre puts a cover file in a directory which contains only one ebook (but multiple formats of the ebook),
this function will return the path of the cover if the ebook_path is in such a situation."""
if not ebook_path.is_file():
raise ValueError(f'Invalid ebook path {ebook_path}')
paths = list(ebook_path.parent.iterdir())
ebook_path_stem, _ = split_path_stem_and_suffix(ebook_path)
cover = None
for path in paths:
stem, suffix = split_path_stem_and_suffix(path)
if path.is_dir():
# We don't care about sub directories.
continue
if stem == ebook_path_stem:
# This is one of the formats of the ebook.
continue
if path.name == 'metadata.opf':
# Metadata, this is expected in a Calibre book directory.
continue
if stem == 'cover' and get_mimetype(path).startswith('image/'):
# Discovered the cover.
cover = path
continue
# Some file that does not match the ebook stem, metadata, or cover. This must not be a Calibre book directory.
return None
return cover
class EBook(ModelHelper, Base):
__tablename__ = 'ebook'
__table_args__ = (
Index('ebook_size_idx', 'size'),
)
id = Column(Integer, primary_key=True)
size: int = Column(Integer)
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
file_group: FileGroup = relationship('FileGroup')
def __repr__(self):
path = str(self.file_group.primary_path)
return f'<EBook id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
def __json__(self) -> dict:
# Convert this eBook's paths to pathlib.Paths, resolving relative to directory.
d = self.file_group.__json__()
if d.get('data') and d['data'].get('ebook_path'):
d['data']['ebook_path'] = self.file_group.resolve_path(d['data']['ebook_path'])
if d.get('data') and d['data'].get('cover_path'):
d['data']['cover_path'] = self.file_group.resolve_path(d['data']['cover_path'])
return d
@staticmethod
def can_model(file_group: FileGroup) -> bool:
if mimetype_is_ebook(file_group.mimetype):
return True
return False
@staticmethod
def do_model(session: Session, file_group: FileGroup) -> 'EBook':
ebook = model_ebook(file_group, session)
file_group.indexed = True
return ebook
def generate_cover(self) -> Optional[pathlib.Path]:
"""Discover this ebook's cover, if found, store it next to the ebook file."""
path = self.file_group.primary_path
if not path:
raise ValueError('Cannot generate cover for ebook without ebook file!')
if calibre_cover := discover_calibre_cover(path):
self.hide_calibre_files()
return calibre_cover
cover_bytes = extract_ebook_cover(path)
if cover_bytes:
cover_path = path.with_suffix('.jpeg')
cover_path.write_bytes(cover_bytes)
return cover_path
logger.warning(f'Unable to generate ebook cover for {self.file_group.primary_path}')
@property
def cover_file(self) -> Optional[dict]:
posters = self.file_group.my_poster_files()
return posters[0]
@property
def cover_path(self) -> Optional[pathlib.Path]:
return self.cover_file['path']
def hide_calibre_files(self):
"""Delete FileGroups of the cover/metadata when this is a Calibre ebook directory."""
path = self.file_group.primary_path
if not discover_calibre_cover(path):
raise ValueError(f'Refusing to hide Calibre files when we do not have a Calibre ebook directory!')
metadata = path.parent / 'metadata.opf'
cover = path.parent / 'cover.jpg'
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup) \
.filter(or_(
FileGroup.primary_path == str(metadata),
FileGroup.primary_path == str(cover),
))
for file_group in file_groups:
session.delete(file_group)
def model_ebook(file_group: FileGroup, session: Session) -> EBook:
"""Creates an EBook model based off a File. Searches for its cover in the provided `files`."""
# Check if an EBook already exists for this FileGroup
ebook = session.query(EBook).filter_by(file_group_id=file_group.id).one_or_none()
if not ebook:
# Create new EBook if it doesn't exist
ebook = EBook(file_group=file_group)
session.add(ebook)
# Multiple formats may be available.
epub_file = next(iter(ebook.file_group.my_epub_files()), None)
mobi_file = next(iter(ebook.file_group.my_mobi_files()), None)
# epub is preferred because it can be indexed.
if epub_file:
ebook_file = epub_file['path']
ebook_mimetype = epub_file['mimetype']
elif mobi_file:
ebook_file = mobi_file['path']
ebook_mimetype = mobi_file['mimetype']
else:
raise ValueError(f'No epub or mobi found. {ebook}')
size = ebook_file.stat().st_size
# Only index if it hasn't been done, or if the file has changed.
changed = size != ebook.size or not ebook.file_group.title
if epub_file and changed:
# Only read the contents of the file if it has changed.
try:
data = extract_ebook_data(ebook_file, ebook_mimetype)
if data:
# Title is a_text.
ebook.file_group.title = ebook.file_group.a_text = data.title
# Author is b_text.
ebook.file_group.b_text = ebook.file_group.author = data.author
# All text is d_text.
ebook.file_group.d_text = data.text
d: dict = data.__json__()
# Store ebook_path as filename only (relative to directory)
d['ebook_path'] = ebook_file.name
ebook.file_group.data = d
except Exception as e:
logger.error(f'Failed to extract epub book data', exc_info=e)
if ebook.file_group.my_poster_files():
# Store cover_path as filename only (relative to directory)
cover_path = ebook.file_group.my_poster_files()[0]['path']
ebook.file_group.data['cover_path'] = cover_path.name
else:
# Extract cover because there is no cover file.
cover_path = ebook.generate_cover()
if cover_path:
ebook.file_group.append_files(cover_path)
# Store cover_path as filename only (relative to directory)
ebook.file_group.data['cover_path'] = cover_path.name
if not ebook.file_group.title:
# Book was not indexed above (probably a MOBI), use the file data.
stem, _ = split_path_stem_and_suffix(ebook.file_group.primary_path)
ebook.file_group.title = stem
ebook.size = size
ebook.flush()
return ebook
def find_ebook_files_in_group(group: List):
"""Returns a list of any File records if they are an ebook."""
return list(filter(lambda i: mimetype_is_ebook(i.mimetype), group))
@register_modeler
async def ebook_modeler(progress_callback: Callable[[int], None] = None):
"""Searches for ebook files and models them into the ebook table."""
total_processed = 0
while True:
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup, EBook).filter(
or_(
FileGroup.mimetype == 'application/epub+zip',
FileGroup.mimetype == 'application/x-mobipocket-ebook',
),
FileGroup.indexed == False,
).outerjoin(EBook, and_(EBook.file_group_id == FileGroup.id)) \
.limit(10)
file_groups: List[Tuple[FileGroup, EBook]] = list(file_groups)
processed = 0
for file_group, ebook in file_groups:
processed += 1
try:
if PYTEST:
# Handle caching issue during testing.
session.expire(file_group)
ebook = ebook or model_ebook(file_group, session)
session.add(ebook)
file_group.model = EBook.__tablename__
file_group.indexed = True
except Exception as e:
logger.error(f'Failed to index ebook {file_group}', exc_info=e)
if PYTEST:
raise
session.commit()
# Report batch progress
total_processed += len(file_groups)
if progress_callback and len(file_groups) > 0:
progress_callback(total_processed)
if processed < 10:
# Did not reach limit, no more books.
break
# Sleep to catch cancel.
await asyncio.sleep(0)

View file

@ -8,8 +8,8 @@ from zipfile import ZipFile
import docx
from wrolpi.cmd import CATDOC_PATH, TEXTUTIL_PATH
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
from wrolpi.common import logger, split_lines_by_length, truncate_object_bytes, extract_html_text, get_title_from_html
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
logger = logger.getChild(__name__)

View file

@ -274,7 +274,7 @@ async def delete(*paths: Union[str, pathlib.Path]):
def _mimetype_suffix_map(path: Path, mimetype: str):
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
from wrolpi.files.ebooks import MOBI_MIMETYPE
from modules.docs.models import MOBI_MIMETYPE
suffix = path.suffix.lower()
if mimetype == 'application/octet-stream':
if suffix.endswith('.mobi'):
@ -315,6 +315,12 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
return 'text/srt'
if mimetype == 'application/zip' and suffix == '.docx':
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
if mimetype == 'application/zip' and suffix == '.cbz':
return 'application/x-cbz'
if mimetype == 'application/zip' and suffix == '.odt':
return 'application/vnd.oasis.opendocument.text'
if mimetype in ('application/x-rar', 'application/x-rar-compressed') and suffix == '.cbr':
return 'application/x-cbr'
if mimetype == 'video/x-ms-asf' and suffix == '.wma':
return 'audio/x-ms-wma'
return mimetype
@ -530,6 +536,14 @@ def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]])
if mimetype.startswith('application/x-mobipocket-ebook') and not has_epub:
# Only return a MOBI if no EPUB is present.
return file
if mimetype in ('application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.oasis.opendocument.text',
'application/x-cbz', 'application/x-cbr',
'application/vnd.comicbook+zip', 'application/vnd.comicbook-rar'):
return file
if file.suffix.lower() in ('.cbz', '.cbr', '.cbt', '.cb7'):
return file
# Secondary file types.
for file, mimetype in file_mimetypes:
@ -1440,19 +1454,19 @@ def _bulk_update_file_groups_reorganize(updates: List[dict]):
for update in updates:
# Don't change indexed - reorganization only moves files, content unchanged
curs.execute('''
UPDATE file_group
SET directory = %s,
primary_path = %s,
files = %s,
data = %s
WHERE id = %s
''', (
update['directory'],
update['primary_path'],
json.dumps(update['files'], default=_json_serial),
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
update['id'],
))
UPDATE file_group
SET directory = %s,
primary_path = %s,
files = %s,
data = %s
WHERE id = %s
''', (
update['directory'],
update['primary_path'],
json.dumps(update['files'], default=_json_serial),
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
update['id'],
))
# Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates.
# This is necessary because get_db_curs uses the same connection as get_db_session
@ -2085,12 +2099,16 @@ bulk_tag_worker = perpetual_signal(sleep=1)(bulk_tag_worker)
def _detect_archive_type(path: pathlib.Path) -> str:
"""Detect archive type from file suffix. Returns 'zip', 'tar', or raises UnsupportedArchive."""
"""Detect archive type from file suffix. Returns 'zip', 'tar', 'rar', '7z', or raises UnsupportedArchive."""
name = path.name.lower()
if name.endswith('.zip'):
if name.endswith(('.zip', '.cbz')):
return 'zip'
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz')):
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz', '.cbt')):
return 'tar'
if name.endswith(('.rar', '.cbr')):
return 'rar'
if name.endswith(('.7z', '.cb7')):
return '7z'
raise UnsupportedArchive(f'Unsupported archive format: {path.name}')
@ -2198,6 +2216,36 @@ def list_archive_contents(path: pathlib.Path) -> dict:
if not is_dir:
total_size += member.size
total_files += 1
elif archive_type == 'rar':
import rarfile
with rarfile.RarFile(path, 'r') as rf:
for info in rf.infolist():
is_dir = info.is_dir()
entry = {
'path': info.filename,
'name': info.filename.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': info.file_size,
}
flat_entries.append(entry)
if not is_dir:
total_size += info.file_size
total_files += 1
elif archive_type == '7z':
import py7zr
with py7zr.SevenZipFile(path, 'r') as sz:
for entry_info in sz.list():
is_dir = entry_info.is_directory
entry = {
'path': entry_info.filename + ('/' if is_dir else ''),
'name': entry_info.filename.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': entry_info.uncompressed if not is_dir else 0,
}
flat_entries.append(entry)
if not is_dir:
total_size += entry_info.uncompressed
total_files += 1
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
raise UnsupportedArchive(f'Cannot read archive: {e}')
@ -2247,6 +2295,28 @@ def stream_archive_member(archive_path: pathlib.Path, member_path: str) -> Tuple
if f is None:
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
file_bytes = f.read()
elif archive_type == 'rar':
import rarfile
with rarfile.RarFile(archive_path, 'r') as rf:
try:
info = rf.getinfo(member_path)
except KeyError:
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
if info.is_dir():
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
file_bytes = rf.read(member_path)
elif archive_type == '7z':
import py7zr
import tempfile
with py7zr.SevenZipFile(archive_path, 'r') as sz:
if member_path not in sz.getnames():
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
with tempfile.TemporaryDirectory() as tmpdir:
sz.extract(tmpdir, targets=[member_path])
extracted_path = pathlib.Path(tmpdir) / member_path
if not extracted_path.is_file():
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
file_bytes = extracted_path.read_bytes()
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
raise UnsupportedArchive(f'Cannot read archive: {e}')

View file

@ -1,4 +1,4 @@
import copy
import json
import json
import pathlib
import shutil
@ -98,7 +98,7 @@ class FileGroup(ModelHelper, Base):
indexed = Column(Boolean, default=lambda: False) # wrolpi.files.lib.apply_indexers
length = Column(BigInteger) # video duration, article words, etc.
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
model = Column(String) # "video", "archive", "ebook", etc.
model = Column(String) # "video", "archive", "doc", etc.
modification_datetime = Column(TZDateTime) # the modification date of the file on disk
primary_path: pathlib.Path = Column(MediaPathType, nullable=False, unique=True)
published_datetime = Column(TZDateTime) # the date the creator published this file
@ -551,7 +551,6 @@ class FileGroup(ModelHelper, Base):
This method first checks if the path is a primary_path, and if not,
searches for a FileGroup where the file is in the files list.
"""
from sqlalchemy import text
path = pathlib.Path(path)
directory = str(path.parent)
@ -725,13 +724,13 @@ class FileGroup(ModelHelper, Base):
"""Return the class that is suitable for this FileGroup (usually based on mimetype), if any."""
from modules.videos.models import Video
from modules.archive.models import Archive
from wrolpi.files.ebooks import EBook
from modules.docs.models import Doc
from modules.zim.models import Zim
if Video.can_model(self):
return Video
elif EBook.can_model(self):
return EBook
elif Doc.can_model(self):
return Doc
elif Archive.can_model(self):
return Archive
elif Zim.can_model(self):

View file

@ -1,191 +0,0 @@
import asyncio
import dataclasses
import pathlib
from typing import Callable, Generator
import pytz
from wrolpi import dates
from wrolpi.common import register_modeler, logger, truncate_generator_bytes, truncate_object_bytes, \
split_lines_by_length, slow_logger
from wrolpi.db import get_db_session
from wrolpi.files.indexers import Indexer
from wrolpi.files.models import FileGroup
from wrolpi.vars import FILE_MAX_PDF_SIZE, PYTEST, FILE_MAX_TEXT_SIZE
try:
import pymupdf
except ImportError:
pymupdf = None
logger = logger.getChild(__name__)
__all__ = ['get_pdf_metadata', 'get_words', 'pdf_modeler']
@dataclasses.dataclass
class PDFMetadata:
title: str = None
author: str = None
published_datetime: str = None
modification_datetime: str = None
def _parse_pdf_date(date_str: str):
"""Parse PDF date format 'D:YYYYMMDDHHmmSS+HH'mm'' to datetime."""
if not date_str:
return None
try:
parsed = dates.strpdate(date_str)
if parsed and not parsed.tzinfo:
parsed = parsed.astimezone(pytz.UTC)
return parsed
except Exception:
return None
def get_pdf_metadata(doc, path: pathlib.Path) -> PDFMetadata:
"""Extract title/author from the PDF metadata."""
data = PDFMetadata()
if doc is None or doc.metadata is None:
logger.error(f'Cannot get title of {path} PyMuPDF is not installed.')
data.title = Indexer.get_title(path)
return data
metadata = doc.metadata
if (title := metadata.get('title')) and title.lower() != 'unknown':
data.title = title
if (author := metadata.get('author')) and author.lower() != 'unknown':
data.author = author
if creation_date := metadata.get('creationDate'):
data.published_datetime = _parse_pdf_date(creation_date)
if mod_date := metadata.get('modDate'):
data.modification_datetime = _parse_pdf_date(mod_date)
return data
def get_words(doc, path: pathlib.Path) -> Generator[str, None, None]:
"""
Reads all text in a PDF.
Note, may return more than can be stored in Postgres. See: truncate_generator_bytes
"""
if pymupdf is None:
logger.error(f'Cannot index {path} PyMuPDF is not installed.')
yield ''
return
for page in doc:
text = page.get_text()
# Postgres does not allow null characters.
text = text.replace('\x00', '\uFFFD').strip()
if text:
yield text
# PDFs can be large, limit how many are modeled at once.
PDF_PROCESSING_LIMIT = 10
@register_modeler
async def pdf_modeler(progress_callback: Callable[[int], None] = None):
"""Queries for any PDF files that have not been indexed. Each PDF found will be indexed."""
if not pymupdf:
logger.warning(f'Cannot index PDF without PyMuPDF')
return
total_processed = 0
while True:
# Continually query for PDFs that have not been indexed.
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup) \
.filter(
# Get all groups that contain a PDF that have not been indexed.
FileGroup.indexed != True,
FileGroup.mimetype == 'application/pdf',
).limit(PDF_PROCESSING_LIMIT)
file_groups = list(file_groups)
processed = 0
for file_group in file_groups:
processed += 1
pdf_files = file_group.my_files('application/pdf')
if not pdf_files:
logger.error('Query returned a group without a PDF!')
continue
# Assume the first PDF is the only PDF.
pdf_file = pdf_files[0]['path']
doc = None
try:
file_title = Indexer.get_title(pdf_file)
doc = pymupdf.open(pdf_file)
with slow_logger(2, f'Modeling PDF took %(elapsed)s seconds: {file_group}',
logger__=logger):
metadata = get_pdf_metadata(doc, pdf_file)
words = ''
if pdf_file.stat().st_size > FILE_MAX_PDF_SIZE:
logger.warning(f'PDF too large to fully index: {pdf_file}')
else:
try:
# PDFs are complex, don't fail to create title index if text extraction fails.
words = '\n'.join(
truncate_generator_bytes(get_words(doc, pdf_file), FILE_MAX_TEXT_SIZE))
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
words = split_lines_by_length(words)
except Exception as e:
logger.error(f'Failed to index {pdf_file}', exc_info=e)
if PYTEST:
raise
file_group.title = file_group.a_text = metadata.title or file_title
file_group.author = file_group.b_text = metadata.author
file_group.c_text = file_title # The name of the file may not match the title in the PDF metadata.
file_group.d_text = words or None
file_group.published_datetime = metadata.published_datetime
file_group.published_modified_datetime = metadata.modification_datetime
file_group.model = 'pdf'
# Find an image with the same stem to use as the poster.
# Store as filename only (consistent with other models, enables fast moves/renames).
poster_files = file_group.my_poster_files()
if poster_files:
file_group.data = file_group.data or {}
file_group.data['poster_path'] = poster_files[0]['path'].name
except Exception as e:
logger.error(f'Failed to index PDF {pdf_file}', exc_info=e)
if PYTEST:
raise
finally:
if doc:
doc.close()
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
file_group.indexed = True
# Sleep to catch cancel.
await asyncio.sleep(0)
logger.info(f'pdf_modeler processed {processed} PDFs')
session.commit()
# Report batch progress
total_processed += len(file_groups)
if progress_callback and len(file_groups) > 0:
progress_callback(total_processed)
if processed < PDF_PROCESSING_LIMIT:
# Did not reach limit, do not query again.
break
# Sleep to catch cancel.
await asyncio.sleep(0)

View file

@ -103,5 +103,3 @@ class BulkTagApplyRequest:
@dataclass
class ArchiveContentsRequest:
path: str

View file

@ -8,7 +8,6 @@ import pytest
from modules.videos import Video
from wrolpi.common import get_wrolpi_config
from wrolpi.files import lib
from wrolpi.files.lib import get_mimetype
from wrolpi.files.models import FileGroup, Directory
from wrolpi.tags import TagFile
@ -337,7 +336,7 @@ async def test_file_statistics(test_session, async_client, test_directory, examp
'tagged_files': 0,
'tagged_zims': 0,
'tags_count': 0,
'total_count': 6, # extracted cover + .ffprobe.json
'total_count': 7, # extracted covers (epub + pdf) + .ffprobe.json
'video_count': 1,
'zip_count': 0,
}
@ -696,7 +695,7 @@ async def test_post_upload(test_session, async_client, test_directory, make_file
@pytest.mark.asyncio
async def test_post_upload_video_extracts_ffprobe(test_session, async_client, test_directory, make_files_structure,
make_multipart_form, video_bytes):
make_multipart_form, video_bytes):
"""Uploading a video file should extract and store ffprobe data."""
make_files_structure(['uploads/'])
@ -929,7 +928,7 @@ async def test_rename_file(test_session, test_directory, make_files_structure, a
@pytest.mark.asyncio
async def test_rename_file_with_associated_files_via_api(test_session, test_directory, make_files_structure,
async_client, video_bytes, srt_text, refresh_files):
async_client, video_bytes, srt_text, refresh_files):
"""Renaming a FileGroup via API should rename all associated files.
Regression test: When renaming "example.mp4" to "example 2.mp4" and back via API,
@ -1272,7 +1271,7 @@ async def test_get_file_skip_tracking(test_session, async_client, test_directory
@pytest.mark.asyncio
async def test_get_file_ignored_directory_not_tracked(test_session, async_client, test_directory, make_files_structure,
refresh_files, test_wrolpi_config, await_switches):
refresh_files, test_wrolpi_config, await_switches):
"""Files in ignored directories do not get their viewed timestamp set."""
# Create files in a directory that will be ignored
foo, bar = make_files_structure({'config/settings.txt': 'settings', 'normal/file.txt': 'contents'})

View file

@ -240,3 +240,103 @@ async def test_download_traversal_attack(test_session, async_client, test_direct
_, response = await async_client.get('/api/files/zip/download?path=test.zip&member=../../../etc/passwd')
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'INVALID_ARCHIVE_MEMBER' in response.text
# ---- Comic book archive format tests using real sample files ----
@pytest.mark.asyncio
async def test_list_cbt_contents(test_session, async_client, test_directory, example_cbt):
"""CBT (TAR) archive contents can be listed."""
_, response = await async_client.post('/api/files/zip/contents',
content=json.dumps({'path': example_cbt.name}))
assert response.status_code == HTTPStatus.OK
contents = response.json['contents']
assert contents['total_files'] == 4
@pytest.mark.asyncio
async def test_list_cbt_dir_contents(test_session, async_client, test_directory, example_cbt_dir):
"""CBT (TAR) archive with nested directories can be listed."""
_, response = await async_client.post('/api/files/zip/contents',
content=json.dumps({'path': example_cbt_dir.name}))
assert response.status_code == HTTPStatus.OK
contents = response.json['contents']
assert contents['total_files'] == 4
# Files are in a nested directory.
entry_names = [e['name'] for e in contents['entries']]
assert 'images' in entry_names
@pytest.mark.asyncio
async def test_download_cbt_member(test_session, async_client, test_directory, example_cbt):
"""A single image can be downloaded from a CBT archive."""
_, response = await async_client.get(
f'/api/files/zip/download?path={example_cbt.name}&member=Bobby-Make-Believe_1915__0.jpg')
assert response.status_code == HTTPStatus.OK
assert len(response.body) > 0
assert response.body[:2] == b'\xff\xd8' # JPEG magic bytes
@pytest.mark.asyncio
async def test_list_cbr_contents(test_session, async_client, test_directory, example_cbr):
"""CBR (RAR) archive contents can be listed."""
_, response = await async_client.post('/api/files/zip/contents',
content=json.dumps({'path': example_cbr.name}))
assert response.status_code == HTTPStatus.OK
contents = response.json['contents']
assert contents['total_files'] == 4
@pytest.mark.asyncio
async def test_download_cbr_member(test_session, async_client, test_directory, example_cbr):
"""A single image can be downloaded from a CBR archive."""
_, response = await async_client.get(
f'/api/files/zip/download?path={example_cbr.name}&member=Bobby-Make-Believe_1915__0.jpg')
assert response.status_code == HTTPStatus.OK
assert len(response.body) > 0
assert response.body[:2] == b'\xff\xd8' # JPEG magic bytes
@pytest.mark.asyncio
async def test_list_cb7_contents(test_session, async_client, test_directory, example_cb7):
"""CB7 (7z) archive contents can be listed."""
_, response = await async_client.post('/api/files/zip/contents',
content=json.dumps({'path': example_cb7.name}))
assert response.status_code == HTTPStatus.OK
contents = response.json['contents']
assert contents['total_files'] == 4
@pytest.mark.asyncio
async def test_list_cb7_dir_contents(test_session, async_client, test_directory, example_cb7_dir):
"""CB7 (7z) archive with nested directories can be listed."""
_, response = await async_client.post('/api/files/zip/contents',
content=json.dumps({'path': example_cb7_dir.name}))
assert response.status_code == HTTPStatus.OK
contents = response.json['contents']
assert contents['total_files'] == 4
entry_names = [e['name'] for e in contents['entries']]
assert 'images' in entry_names
@pytest.mark.asyncio
async def test_download_cb7_member(test_session, async_client, test_directory, example_cb7):
"""A single image can be downloaded from a CB7 archive."""
_, response = await async_client.get(
f'/api/files/zip/download?path={example_cb7.name}&member=Bobby-Make-Believe_1915__0.jpg')
assert response.status_code == HTTPStatus.OK
assert len(response.body) > 0
assert response.body[:2] == b'\xff\xd8' # JPEG magic bytes
@pytest.mark.asyncio
async def test_list_cbz_dir_contents(test_session, async_client, test_directory, example_cbz_dir):
"""CBZ (ZIP) archive with nested directories can be listed."""
_, response = await async_client.post('/api/files/zip/contents',
content=json.dumps({'path': example_cbz_dir.name}))
assert response.status_code == HTTPStatus.OK
contents = response.json['contents']
assert contents['total_files'] == 4
entry_names = [e['name'] for e in contents['entries']]
assert 'images' in entry_names

View file

@ -1,74 +0,0 @@
"""Test for ebook_modeler loop to ensure it processes more than one batch.
This test ensures the ebook_modeler can process more than its batch limit (10)
files. If the loop logic has an off-by-one error (like using enumerate() with wrong
comparison), the modeler would only process one batch and exit early.
"""
import shutil
import pytest
from wrolpi.files.ebooks import ebook_modeler, EBook
from wrolpi.files.models import FileGroup
from wrolpi.vars import PROJECT_DIR
# The ebook_modeler uses a hardcoded limit of 10
EBOOK_PROCESSING_LIMIT = 10
@pytest.mark.asyncio
async def test_ebook_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory):
"""
Test that ebook_modeler processes MORE than the batch limit (10) files.
This test creates 15 ebook files and verifies that ebook_modeler
processes all of them, not just the first batch of 10.
This catches off-by-one bugs in the loop logic (e.g., using enumerate()
which is 0-indexed but comparing against the limit incorrectly).
"""
ebook_dir = test_directory / 'ebooks'
ebook_dir.mkdir(parents=True)
# Create more ebooks than the batch limit
num_ebooks = EBOOK_PROCESSING_LIMIT + 5 # 15 ebooks
ebook_paths = []
for i in range(num_ebooks):
ebook_path = ebook_dir / f'test_ebook_{i:03d}.epub'
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', ebook_path)
ebook_paths.append(ebook_path)
# Create FileGroups for each ebook file (simulating what refresh does)
for ebook_path in ebook_paths:
fg = FileGroup.from_paths(test_session, ebook_path)
# EPUB mimetype might be 'application/epub+zip' or 'application/epub'
assert fg.mimetype.startswith('application/epub')
test_session.commit()
# Verify we have the expected number of unindexed ebook FileGroups
unindexed_count = test_session.query(FileGroup).filter(
FileGroup.indexed == False,
FileGroup.mimetype == 'application/epub+zip',
).count()
assert unindexed_count == num_ebooks, f"Expected {num_ebooks} unindexed ebook files, got {unindexed_count}"
# Run the ebook_modeler
await ebook_modeler()
# Count how many EBooks were created
ebook_count = test_session.query(EBook).count()
# All ebooks should be created
assert ebook_count == num_ebooks, \
f"ebook_modeler should process ALL {num_ebooks} files, but only processed {ebook_count}. " \
f"This may be an off-by-one bug in the loop logic!"
# Also verify all FileGroups are now indexed
still_unindexed = test_session.query(FileGroup).filter(
FileGroup.indexed == False,
FileGroup.mimetype == 'application/epub+zip',
).count()
assert still_unindexed == 0, \
f"All ebook FileGroups should be indexed, but {still_unindexed} remain unindexed"

View file

@ -1,175 +0,0 @@
import json
import shutil
from http import HTTPStatus
import pytest
from wrolpi.files.ebooks import EBook, EPUB_MIMETYPE
from wrolpi.files.models import FileGroup
from wrolpi.files.worker import file_worker
from wrolpi.test.common import assert_dict_contains
@pytest.mark.asyncio
async def test_index(async_client, test_session, test_directory, example_epub, example_mobi, refresh_files):
"""Ebooks can be indexed.
Covers can be discovered."""
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
assert ebook.file_group.title == 'WROLPi Test Book'
assert ebook.file_group.mimetype.startswith(EPUB_MIMETYPE)
assert ebook.file_group.author == 'roland'
assert ebook.size == 292579
# data now stores relative filenames only (not absolute paths)
assert ebook.file_group.data == {'author': 'roland', 'title': 'WROLPi Test Book',
'cover_path': 'ebook example.jpeg',
'ebook_path': 'ebook example.epub',
}
assert ebook.file_group.a_text, 'Book title was not indexed'
assert ebook.file_group.b_text, 'Book author was not indexed'
assert ebook.file_group.d_text, 'Book text was not indexed'
# Both ebooks are assumed to be the same book, but different formats.
assert (epubs := ebook.file_group.my_files('application/epub+zip')) and len(epubs) == 1 \
and epubs[0]['path'] == example_epub
assert (mobis := ebook.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1 \
and mobis[0]['path'] == example_mobi
# Cover was discovered.
assert len(ebook.file_group.my_poster_files()) == 1
# EPUB, MOBI, JPEG.
assert len(ebook.file_group.files) == 3
# Ebooks can be deleted during refresh.
example_epub.unlink()
await refresh_files()
assert test_session.query(EBook).count() == 1
ebook = test_session.query(EBook).one()
assert not ebook.file_group.my_files('application/epub+zip')
assert (mobis := ebook.file_group.my_files('application/x-mobipocket-ebook')) and len(mobis) == 1 \
and mobis[0]['path'] == example_mobi
# Cover was discovered.
assert len(ebook.file_group.my_poster_files()) == 1
# MOBI, JPEG.
assert len(ebook.file_group.files) == 2
@pytest.mark.asyncio
async def test_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory, await_switches,
refresh_files):
cover_path = example_epub.with_suffix('.jpg')
cover_path.write_bytes(image_bytes_factory())
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
# Cover file near the eBook was discovered.
assert ebook.cover_path.read_bytes() == cover_path.read_bytes()
@pytest.mark.asyncio
async def test_extract_cover(test_session, test_directory, example_epub, await_switches, refresh_files):
"""First image is extracted from the Ebook and used as the cover."""
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
# The WROLPi logo is the first image in the example epub.
assert ebook.cover_path and ebook.cover_path.stat().st_size == 297099
@pytest.mark.asyncio
async def test_search_ebooks(test_session, async_client, example_epub):
"""Ebooks are handled in File search results."""
request, response = await async_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(EBook).count() == 1
ebook: EBook = test_session.query(EBook).one()
assert ebook.file_group.title == 'WROLPi Test Book'
assert ebook.file_group.author == 'roland'
assert ebook.file_group.a_text == 'WROLPi Test Book', 'Book title was not updated'
assert ebook.file_group.d_text, 'Book was not indexed'
content = dict(mimetypes=['application/epub', 'application/x-mobipocket-ebook'])
request, response = await async_client.post('/api/files/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json
file_group = response.json['file_groups'][0]
epub_file = file_group['files'][0]
assert epub_file['path'] == 'ebook example.epub' and epub_file['mimetype'] == 'application/epub+zip'
assert_dict_contains(
file_group['data'],
{'cover_path': 'ebook example.jpeg', 'ebook_path': 'ebook example.epub', 'title': 'WROLPi Test Book',
'author': 'roland'},
)
# No Mobi ebook.
content = dict(mimetypes=['application/x-mobipocket-ebook'])
request, response = await async_client.post('/api/files/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json
assert len(response.json['file_groups']) == 0
@pytest.mark.asyncio
async def test_discover_calibre_cover(test_session, async_client, test_directory, example_epub, example_mobi,
image_file, refresh_files):
"""Calibre puts a cover near an ebook file, test if it can be found."""
# Create a Calibre metadata file.
metadata = test_directory / 'metadata.opf'
metadata.touch()
# Create a "cover.jpg" file near the ebook, this is the Calibre cover.
cover_image = test_directory / 'cover.jpg'
shutil.move(image_file, cover_image)
# Calibre cover image was discovered, no cover was generated.
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
assert ebook.file_group.title == 'WROLPi Test Book'
assert ebook.cover_file['path'] == test_directory / 'cover.jpg'
# Use my_files() to get resolved Path objects
assert {i['path'].name for i in ebook.file_group.my_files()} == {'ebook example.epub', 'ebook example.mobi', 'cover.jpg'}
# Cover is not the first image from the EPUB.
assert ebook.cover_path.stat().st_size != 292579
# Metadata and cover were deleted from the FileGroups.
assert test_session.query(FileGroup).count() == 1
@pytest.mark.asyncio
async def test_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory,
refresh_files, await_background_tasks):
"""An ebook is re-indexed when moved."""
tag = await tag_factory()
shutil.move(image_file, example_epub.with_suffix('.jpg'))
await refresh_files()
ebook: EBook = test_session.query(EBook).one()
assert ebook.cover_path
assert_dict_contains(ebook.file_group.data, dict(author='roland'))
# Use my_files() to get resolved absolute Path objects
assert sorted([i['path'] for i in ebook.file_group.my_files()]) == [
test_directory / 'ebook example.epub',
test_directory / 'ebook example.jpg',
]
ebook.file_group.add_tag(test_session, tag.id)
test_session.commit()
new_directory = test_directory / 'new'
new_directory.mkdir()
file_worker.queue_move(new_directory, [ebook.file_group.primary_path])
await await_background_tasks()
test_session.expire_all()
ebook: EBook = test_session.query(EBook).one()
assert ebook.file_group.primary_path == test_directory / 'new/ebook example.epub'
assert ebook.cover_path
assert_dict_contains(ebook.file_group.data, dict(author='roland'))
# Use my_files() to get resolved absolute Path objects
assert sorted([i['path'] for i in ebook.file_group.my_files()]) == [
test_directory / 'new/ebook example.epub',
test_directory / 'new/ebook example.jpg',
]

View file

@ -158,7 +158,7 @@ async def test_delete_similar_names(async_client, test_session, make_files_struc
])
await lib.delete('config/._wrolpi.yaml', 'config/._wrolpi.yaml.sb-98fa34a1-MG7phT',
'config/._wrolpi.yaml.sb-98fa34a1-N4k8NB')
'config/._wrolpi.yaml.sb-98fa34a1-N4k8NB')
@pytest.mark.parametrize(
@ -948,7 +948,7 @@ async def test_doc_indexer(async_client, test_session, example_doc, refresh_file
await refresh_files()
assert test_session.query(FileGroup).count() == 1
doc = test_session.query(FileGroup).one()
assert doc.title == 'example word.doc'
assert doc.title == 'example word doc'
assert doc.a_text == 'example word doc'
if IS_MACOS:
assert doc.d_text == 'Example Word Document\nSecond page'
@ -962,8 +962,9 @@ async def test_docx_indexer(async_client, test_session, example_docx, refresh_fi
await refresh_files()
assert test_session.query(FileGroup).count() == 1
doc = test_session.query(FileGroup).one()
assert doc.title == 'example word.docx'
assert doc.a_text == 'example word docx'
# The docx file has a "Word Document" title in its core properties.
assert doc.title == 'Word Document'
assert doc.a_text == 'Word Document'
assert doc.d_text == 'Example Word Document Second page'

View file

@ -132,5 +132,3 @@ def test_file_group_update_wrolpi_json_noop_no_metadata_json(test_session, make_
fg.update_wrolpi_json({'key': 'value'})
assert not (get_media_directory() / 'video.info.json').exists()
assert not (get_media_directory() / 'video.readability.json').exists()

View file

@ -1,73 +0,0 @@
"""Test for pdf_modeler loop to ensure it processes more than one batch.
This test ensures the pdf_modeler can process more than PDF_PROCESSING_LIMIT (10)
files. If the loop logic has an off-by-one error (like using enumerate() with wrong
comparison), the modeler would only process one batch and exit early.
"""
import shutil
import pytest
from wrolpi.files.pdfs import pdf_modeler, PDF_PROCESSING_LIMIT
from wrolpi.files.models import FileGroup
from wrolpi.vars import PROJECT_DIR
@pytest.mark.asyncio
async def test_pdf_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory):
"""
Test that pdf_modeler processes MORE than PDF_PROCESSING_LIMIT (10) files.
This test creates 15 PDF files and verifies that pdf_modeler
processes all of them, not just the first batch of 10.
This catches off-by-one bugs in the loop logic (e.g., using enumerate()
which is 0-indexed but comparing against the limit incorrectly).
"""
pdf_dir = test_directory / 'pdfs'
pdf_dir.mkdir(parents=True)
# Create more PDFs than the batch limit
num_pdfs = PDF_PROCESSING_LIMIT + 5 # 15 PDFs
pdf_paths = []
for i in range(num_pdfs):
pdf_path = pdf_dir / f'test_pdf_{i:03d}.pdf'
shutil.copy(PROJECT_DIR / 'test/pdf example.pdf', pdf_path)
pdf_paths.append(pdf_path)
# Create FileGroups for each PDF file (simulating what refresh does)
for pdf_path in pdf_paths:
fg = FileGroup.from_paths(test_session, pdf_path)
assert fg.mimetype == 'application/pdf'
test_session.commit()
# Verify we have the expected number of unindexed PDF FileGroups
unindexed_count = test_session.query(FileGroup).filter(
FileGroup.indexed != True,
FileGroup.mimetype == 'application/pdf',
).count()
assert unindexed_count == num_pdfs, f"Expected {num_pdfs} unindexed PDF files, got {unindexed_count}"
# Run the pdf_modeler
await pdf_modeler()
# Count how many FileGroups are now indexed as PDFs
indexed_pdf_count = test_session.query(FileGroup).filter(
FileGroup.indexed == True,
FileGroup.model == 'pdf',
).count()
# All PDFs should be indexed
assert indexed_pdf_count == num_pdfs, \
f"pdf_modeler should process ALL {num_pdfs} files, but only processed {indexed_pdf_count}. " \
f"This may be an off-by-one bug in the loop logic!"
# Also verify no FileGroups remain unindexed
still_unindexed = test_session.query(FileGroup).filter(
FileGroup.indexed != True,
FileGroup.mimetype == 'application/pdf',
).count()
assert still_unindexed == 0, \
f"All PDF FileGroups should be indexed, but {still_unindexed} remain unindexed"

View file

@ -1,94 +0,0 @@
from unittest import mock
import pytest
from PIL import Image
from wrolpi.dates import now
from wrolpi.files import lib as files_lib
from wrolpi.files import pdfs
from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_pdf_title(async_client, test_session, example_pdf, refresh_files):
"""PDF modeler extracts a PDFs title and contents."""
await refresh_files()
with mock.patch('wrolpi.files.pdfs.get_pdf_metadata') as mock_get_pdf_metadata:
mock_get_pdf_metadata.side_effect = Exception('Should not index twice')
await refresh_files()
pdf: FileGroup = test_session.query(FileGroup).one()
assert pdf.indexed is True
assert pdf.title == pdf.a_text == 'WROLPi Test PDF'
assert pdf.b_text == 'roland'
assert pdf.c_text == 'pdf example pdf'
assert pdf.d_text and pdf.d_text.startswith('Page one\n') and len(pdf.d_text) == 467
# Metadata within the PDF.
assert pdf.author
assert pdf.published_datetime
# The date the file was modified.
assert pdf.modification_datetime
@pytest.mark.asyncio
async def test_pdf_modeler(test_session, example_pdf):
"""PDFs can be indexed by PDFIndexer."""
files_lib._upsert_files([example_pdf], now())
await pdfs.pdf_modeler()
file_group = test_session.query(FileGroup).one()
a_text, b_text, c_text, d_text = file_group.a_text, file_group.b_text, file_group.c_text, file_group.d_text
# The title extracted from the PDF.
assert a_text == 'WROLPi Test PDF'
# The author.
assert b_text == 'roland'
# The parsed file name.
assert c_text == 'pdf example pdf'
# All pages are extracted. Text is formatted to fit on vertical screen.
assert d_text == 'Page one\n' \
'Page two\n' \
'Lorem ipsum dolor sit amet,\n' \
'consectetur adipiscing elit, sed do\n' \
'eiusmod tempor incididunt ut labore et\n' \
'\n' \
'dolore magna aliqua. Ut enim ad minim\n' \
'veniam, quis nostrud exercitation\n' \
'ullamco laboris nisi ut \n' \
'aliquip ex ea commodo consequat. Duis\n' \
'aute irure dolor in reprehenderit in\n' \
'voluptate velit esse cillum \n' \
'dolore eu fugiat nulla pariatur.\n' \
'Excepteur sint occaecat cupidatat non\n' \
'proident, sunt in culpa qui officia \n' \
'deserunt mollit anim id est laborum.'
@pytest.mark.asyncio
async def test_pdf_indexer_max_size(test_session, example_pdf):
"""The contents of a large PDF are not indexed."""
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
files_lib._upsert_files([example_pdf], now())
await pdfs.pdf_modeler()
file_group = test_session.query(FileGroup).one()
assert file_group.d_text is None
@pytest.mark.asyncio
async def test_pdf_poster(test_session, example_pdf):
"""PDF modeler finds an image with the same stem and uses it as the poster."""
# Create an image with the same stem as the PDF.
poster_path = example_pdf.with_suffix('.jpg')
Image.new('RGB', (25, 25), color='grey').save(poster_path)
files_lib._upsert_files([example_pdf, poster_path], now())
await pdfs.pdf_modeler()
file_group = test_session.query(FileGroup).one()
assert file_group.indexed is True
assert file_group.data is not None
# data['poster_path'] stores filename only (for fast moves/renames).
assert file_group.data.get('poster_path') == poster_path.name

View file

@ -473,7 +473,8 @@ async def test_file_worker_status_during_refresh(async_client, test_session, tes
@pytest.mark.asyncio
async def test_file_worker_status_resets_after_refresh(async_client, test_session, test_directory, make_files_structure):
async def test_file_worker_status_resets_after_refresh(async_client, test_session, test_directory,
make_files_structure):
"""FileWorker resets status to idle after refresh completes."""
make_files_structure(['docs/file1.txt'])
@ -509,7 +510,7 @@ async def test_file_worker_count_only_resets_status(async_client, test_session,
@pytest.mark.asyncio
async def test_file_worker_refresh_inserts_new_files(async_client, test_session, test_directory, make_files_structure,
video_file_factory):
video_file_factory):
"""FileWorker inserts new FileGroups during refresh."""
# Create files on disk but not in DB
# Note: txt/json files without recognizable primary types (video, epub, etc.)
@ -898,7 +899,8 @@ async def test_handle_refresh_runs_modelers(async_client, test_session, test_dir
@pytest.mark.asyncio
async def test_handle_refresh_status_includes_indexing(async_client, test_session, test_directory, make_files_structure):
async def test_handle_refresh_status_includes_indexing(async_client, test_session, test_directory,
make_files_structure):
"""FileWorker.handle_refresh should update status during indexing phase."""
# Create files
make_files_structure(['file1.txt', 'file2.txt'])
@ -926,7 +928,8 @@ async def test_handle_refresh_status_includes_indexing(async_client, test_sessio
@pytest.mark.asyncio
async def test_refresh_complete_flag_set_on_full_refresh(async_client, test_session, test_directory, make_files_structure, flags_lock):
async def test_refresh_complete_flag_set_on_full_refresh(async_client, test_session, test_directory,
make_files_structure, flags_lock):
"""refresh_complete flag is set when refreshing the entire media directory."""
from wrolpi import flags
@ -946,7 +949,8 @@ async def test_refresh_complete_flag_set_on_full_refresh(async_client, test_sess
@pytest.mark.asyncio
async def test_refresh_complete_flag_not_set_on_partial_refresh(async_client, test_session, test_directory, make_files_structure, flags_lock):
async def test_refresh_complete_flag_not_set_on_partial_refresh(async_client, test_session, test_directory,
make_files_structure, flags_lock):
"""refresh_complete flag is NOT set when refreshing only a subdirectory."""
from wrolpi import flags
@ -1251,7 +1255,7 @@ async def test_file_worker_refresh_file_skips_count_phase(
@pytest.mark.asyncio
async def test_queue_refresh_returns_job_id(async_client, test_session, test_directory, make_files_structure,
await_background_tasks):
await_background_tasks):
"""queue_refresh returns a job_id that can be tracked to completion."""
make_files_structure(['docs/file1.txt'])
@ -1270,7 +1274,7 @@ async def test_queue_refresh_returns_job_id(async_client, test_session, test_dir
@pytest.mark.asyncio
async def test_wait_for_job(async_client, test_session, test_directory, make_files_structure,
await_background_tasks):
await_background_tasks):
"""wait_for_job waits for a job to complete."""
make_files_structure(['docs/file1.txt'])
@ -1291,7 +1295,7 @@ async def test_wait_for_job(async_client, test_session, test_directory, make_fil
@pytest.mark.asyncio
async def test_file_worker_counting_flag_is_set(async_client, test_session, test_directory, make_files_structure,
flags_lock):
flags_lock):
"""file_worker_counting flag is set during counting phase."""
from wrolpi import flags
@ -1320,7 +1324,7 @@ async def test_file_worker_counting_flag_is_set(async_client, test_session, test
@pytest.mark.asyncio
async def test_file_worker_discovery_flag_is_set(async_client, test_session, test_directory, make_files_structure,
flags_lock):
flags_lock):
"""file_worker_discovery flag is set during discovery phases (comparing, upserting, deleting)."""
from wrolpi import flags
@ -1362,7 +1366,7 @@ async def test_upsert_files_progress_callback(test_session, test_directory, make
@pytest.mark.asyncio
async def test_upsert_file_groups_tracks_actual_db_progress(async_client, test_session, test_directory,
make_files_structure):
make_files_structure):
"""_upsert_file_groups tracks progress of actual DB operations, not just diff collection."""
make_files_structure(['docs/file1.txt', 'docs/file2.txt', 'docs/file3.txt'])

View file

@ -1392,7 +1392,7 @@ class FileWorker:
logger.info(f'Deleted {deleted_count} FileGroups ({len(all_urls)} URLs added to skip list)')
def _cleanup_modified_models(self, diffs: list[FileGroupDiff]):
"""Delete Videos/Archives/EBooks for modified FileGroups where primary file was removed.
"""Delete Videos/Archives/Docs for modified FileGroups where primary file was removed.
Optimized: uses a single UNION ALL query to fetch all model types at once,
reducing 3 SELECT queries to 1.
@ -1421,16 +1421,16 @@ class FileWorker:
JOIN file_group fg ON a.file_group_id = fg.id
WHERE fg.id = ANY (%s)
UNION ALL
SELECT 'ebook', e.id, fg.id, fg.primary_path
FROM ebook e
JOIN file_group fg ON e.file_group_id = fg.id
SELECT 'doc', d.id, fg.id, fg.primary_path
FROM doc d
JOIN file_group fg ON d.file_group_id = fg.id
WHERE fg.id = ANY (%s)
''', (file_group_ids, file_group_ids, file_group_ids))
# Group results by model type
video_ids_to_delete = []
archive_ids_to_delete = []
ebook_ids_to_delete = []
doc_ids_to_delete = []
for model_type, model_id, fg_id, primary_path in curs.fetchall():
basename = pathlib.Path(primary_path).name
@ -1439,8 +1439,8 @@ class FileWorker:
video_ids_to_delete.append(model_id)
elif model_type == 'archive':
archive_ids_to_delete.append(model_id)
elif model_type == 'ebook':
ebook_ids_to_delete.append(model_id)
elif model_type == 'doc':
doc_ids_to_delete.append(model_id)
# Batch delete each model type
if video_ids_to_delete:
@ -1461,14 +1461,14 @@ class FileWorker:
''', (archive_ids_to_delete,))
curs.execute('DELETE FROM archive WHERE id = ANY(%s)', (archive_ids_to_delete,))
if ebook_ids_to_delete:
logger.info(f'Deleting {len(ebook_ids_to_delete)} EBooks whose primary file was removed')
if doc_ids_to_delete:
logger.info(f'Deleting {len(doc_ids_to_delete)} Docs whose primary file was removed')
curs.execute('''
UPDATE file_group
SET model = NULL
WHERE id IN (SELECT file_group_id FROM ebook WHERE id = ANY (%s))
''', (ebook_ids_to_delete,))
curs.execute('DELETE FROM ebook WHERE id = ANY(%s)', (ebook_ids_to_delete,))
WHERE id IN (SELECT file_group_id FROM doc WHERE id = ANY (%s))
''', (doc_ids_to_delete,))
curs.execute('DELETE FROM doc WHERE id = ANY(%s)', (doc_ids_to_delete,))
def _validate_move_paths(
self,

View file

@ -13,6 +13,7 @@ from vininfo import Vin
from vininfo.details._base import VinDetails
from modules.archive.api import archive_bp
from modules.docs.api import docs_bp
from modules.inventory import inventory_bp
from modules.map.api import map_bp
from modules.otp.api import otp_bp
@ -49,6 +50,7 @@ api_app.blueprint(api_bp)
api_app.blueprint(archive_bp)
api_app.blueprint(collection_bp) # Unified collection endpoints
api_app.blueprint(config_bp)
api_app.blueprint(docs_bp)
api_app.blueprint(files_bp)
api_app.blueprint(inventory_bp)
api_app.blueprint(map_bp)
@ -710,16 +712,21 @@ async def post_upgrade_start(_: Request):
async def post_search_suggestions(request: Request, body: schema.SearchSuggestionsRequest):
from modules.videos.channel.lib import search_channels_by_name
from modules.archive.lib import search_domains_by_name
from modules.docs.lib import search_authors_by_name, search_subjects_by_name
session = request.ctx.session
channels, domains = await asyncio.gather(
channels, domains, authors, subjects = await asyncio.gather(
search_channels_by_name(session, body.search_str, order_by_video_count=body.order_by_video_count),
search_domains_by_name(session, body.search_str),
search_authors_by_name(session, body.search_str),
search_subjects_by_name(session, body.search_str),
)
ret = dict(
channels=channels,
domains=domains,
authors=authors,
subjects=subjects,
)
return json_response(ret)

View file

@ -152,5 +152,6 @@ async def await_switches(timeout: int = 8):
# All switches handled.
break
else:
logger.error(f'await_switches: TIMEOUT after {count} iterations, switches={list(api_app.shared_ctx.switches.keys())}, changed={api_app.shared_ctx.switches_changed.is_set()}')
logger.error(
f'await_switches: TIMEOUT after {count} iterations, switches={list(api_app.shared_ctx.switches.keys())}, changed={api_app.shared_ctx.switches_changed.is_set()}')
raise RuntimeError('Timed out waiting for switches. Did you remember to use the `await_switches` fixture?')

View file

@ -1,4 +1,3 @@
import contextlib
import pathlib
from dataclasses import dataclass, field
@ -517,7 +516,8 @@ class TagsConfig(ConfigFile):
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
backup_file = self._get_backup_file(backup_date)
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(tags={}, tag_files=[], tag_zims=[])
current_data = self.read_config_file() if self.get_file().is_file() else dict(tags={}, tag_files=[],
tag_zims=[])
backup_tags = backup_data.get('tags', {})
current_tags = current_data.get('tags', {})
@ -586,7 +586,8 @@ class TagsConfig(ConfigFile):
shutil.copy2(backup_file, config_file)
elif mode == 'merge':
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(tags={}, tag_files=[], tag_zims=[], version=0)
current_data = self.read_config_file() if config_file.is_file() else dict(tags={}, tag_files=[],
tag_zims=[], version=0)
# Merge tags: add missing tags only
merged_tags = dict(current_data.get('tags', {}))
@ -791,10 +792,10 @@ def get_tags() -> List[dict]:
SELECT t.id,
t.name,
t.color,
(SELECT COUNT(*) FROM tag_file WHERE tag_id = t.id) AS file_group_count,
(SELECT COUNT(*) FROM tag_zim WHERE tag_id = t.id) AS zim_entry_count,
(SELECT COUNT(*) FROM tag_file WHERE tag_id = t.id) AS file_group_count,
(SELECT COUNT(*) FROM tag_zim WHERE tag_id = t.id) AS zim_entry_count,
(SELECT COUNT(*) FROM collection WHERE tag_id = t.id AND kind = 'channel') AS channel_count,
(SELECT COUNT(*) FROM collection WHERE tag_id = t.id AND kind = 'domain') AS domain_count
(SELECT COUNT(*) FROM collection WHERE tag_id = t.id AND kind = 'domain') AS domain_count
FROM tag t
GROUP BY t.id, t.name, t.color
ORDER BY t.name
@ -806,15 +807,18 @@ def get_tags() -> List[dict]:
def get_recent_tags(limit: int = 5) -> List[str]:
with get_db_curs() as curs:
curs.execute('''
SELECT t.name FROM (
SELECT tag_id, MAX(created_at) AS latest FROM (
SELECT tag_id, created_at FROM tag_file
UNION ALL
SELECT tag_id, created_at FROM tag_zim
) AS combined
GROUP BY tag_id ORDER BY latest DESC LIMIT %(limit)s
) AS recent
JOIN tag t ON t.id = recent.tag_id ORDER BY recent.latest DESC
SELECT t.name
FROM (SELECT tag_id, MAX(created_at) AS latest
FROM (SELECT tag_id, created_at
FROM tag_file
UNION ALL
SELECT tag_id, created_at
FROM tag_zim) AS combined
GROUP BY tag_id
ORDER BY latest DESC
LIMIT %(limit)s) AS recent
JOIN tag t ON t.id = recent.tag_id
ORDER BY recent.latest DESC
''', dict(limit=limit))
return [row['name'] for row in curs.fetchall()]

View file

@ -1,5 +1,5 @@
version: 0
skip_urls: []
skip_urls: [ ]
downloads:
- url: 'https://example.com/channel'
downloader: video

View file

@ -1,5 +1,5 @@
version: 0
skip_urls: []
skip_urls: [ ]
downloads:
- url: 'https://example.com/minimal'
downloader: video
@ -8,4 +8,4 @@ downloads:
destination: null
sub_downloader: null
settings: null
tag_names: []
tag_names: [ ]

View file

@ -123,7 +123,7 @@ class TestTagsBackupImport:
@pytest.mark.asyncio
async def test_import_merge(self, test_directory, test_tags_config, test_session, async_client):
from wrolpi.tags import get_tags_config, Tag
from wrolpi.tags import get_tags_config
config = get_tags_config()
# Write current config with one tag
@ -148,7 +148,7 @@ class TestTagsBackupImport:
@pytest.mark.asyncio
async def test_import_overwrite(self, test_directory, test_tags_config, test_session, async_client):
from wrolpi.tags import get_tags_config, Tag
from wrolpi.tags import get_tags_config
config = get_tags_config()
# Write current config with two tags
@ -641,7 +641,6 @@ class TestImportBackupPreservesCurrentConfig:
assert preserved['version'] == 5
class TestConfigStatusIncludesBackupInfo:
def test_db_config_has_backup_info(self, test_directory, test_tags_config):
@ -678,7 +677,6 @@ class TestNotImplementedForNonDBConfigs:
class TestBackupAPIEndpoints:
async def test_get_backups(self, async_client, test_directory, test_tags_config):
from wrolpi.tags import get_tags_config
backup_dir = test_directory / 'config' / 'backup'
backup_dir.mkdir(parents=True, exist_ok=True)
(backup_dir / 'tags-20260301.yaml').write_text('tags: {}\nversion: 1')
@ -881,7 +879,8 @@ class TestPreviewWithRealisticData:
current = dict(
downloads=[
dict(url='https://example.com', downloader='video', frequency=None, status='failed'),
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000, status='complete'),
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000,
status='complete'),
],
skip_urls=['https://skip1.com'],
version=260,
@ -892,7 +891,8 @@ class TestPreviewWithRealisticData:
backup = dict(
downloads=[
dict(url='https://example.com', downloader='video', frequency=None, status='failed'),
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000, status='complete'),
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000,
status='complete'),
dict(url='https://youtube.com/watch?v=TEST', downloader='video', frequency=None, status='new'),
],
skip_urls=['https://skip1.com'],
@ -914,7 +914,8 @@ class TestPreviewWithRealisticData:
current = dict(
downloads=[
dict(url='https://example.com', downloader='video', frequency=None, status='failed'),
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000, status='complete'),
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000,
status='complete'),
],
skip_urls=[],
version=260,
@ -924,7 +925,8 @@ class TestPreviewWithRealisticData:
backup = dict(
downloads=[
dict(url='https://example.com', downloader='video', frequency=None, status='failed'),
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000, status='complete'),
dict(url='https://youtube.com/channel/ABC', downloader='video_channel', frequency=2592000,
status='complete'),
dict(url='https://youtube.com/watch?v=TEST', downloader='video', frequency=None, status='new'),
],
skip_urls=[],

View file

@ -2,7 +2,7 @@ import asyncio
import json
import pathlib
from abc import ABC
from datetime import datetime, time, timezone, timedelta
from datetime import datetime, time, timezone
from http import HTTPStatus
from itertools import zip_longest
from unittest import mock
@ -16,7 +16,7 @@ from wrolpi.dates import Seconds
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \
set_download_progress, clear_download_progress, make_progress_callback, download_manager
set_download_progress, clear_download_progress, make_progress_callback
from wrolpi.errors import InvalidDownload, WROLModeEnabled
from wrolpi.test.common import assert_dict_contains
@ -1196,7 +1196,6 @@ def test_is_within_download_window_uses_local_time(test_download_manager):
def test_get_local_time_with_configured_timezone(test_download_manager):
"""When config.timezone is set, _get_local_time() converts to that timezone."""
from zoneinfo import ZoneInfo
config = get_wrolpi_config()
config.timezone = 'America/Denver'

View file

@ -295,7 +295,7 @@ class TestDownloadManagerConfigEdgeCases:
assert downloads[0].url == 'https://example.com/new'
async def test_import_config_with_missing_optional_fields(self, test_session, test_directory,
test_download_manager_config, async_client):
test_download_manager_config, async_client):
"""
Config files from older versions or manual edits may be missing optional fields.
Import should succeed with None values for missing fields.
@ -331,7 +331,7 @@ class TestDownloadManagerConfigEdgeCases:
assert downloads[0].next_download is None
async def test_update_existing_download_with_missing_optional_fields(self, test_session, test_directory,
test_download_manager_config, async_client):
test_download_manager_config, async_client):
"""
When updating an existing download from config, missing optional fields should not cause errors.
"""

View file

@ -10,7 +10,7 @@ from wrolpi.api_utils import json_error_handler
from wrolpi.common import get_wrolpi_config
from wrolpi.downloader import Download, get_download_manager_config
from wrolpi.errors import ValidationError, SearchEmpty
from wrolpi.test.common import skip_circleci, assert_dict_contains
from wrolpi.test.common import assert_dict_contains
@pytest.mark.asyncio

View file

@ -248,7 +248,8 @@ async def test_rss_downloader_normalizes_youtube_shorts_urls(test_session, test_
@pytest.mark.asyncio
async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_session, test_download_manager, await_switches):
async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_session, test_download_manager,
await_switches):
"""RSS downloader should normalize YouTube Shorts URLs before checking the skip list.
If a video was skipped with a normalized URL, the RSS feed returning a shorts URL should

View file

@ -548,7 +548,7 @@ async def test_tags_directory_file_replaced(test_session, test_directory, tag_fa
@pytest.mark.asyncio
async def test_tags_directory_partial_file_deleted(test_session, test_directory, tag_factory, video_bytes,
image_bytes_factory, await_switches):
image_bytes_factory, await_switches):
"""When one file in a FileGroup is deleted, its hardlink should be removed."""
from wrolpi import tags as tags_module
@ -584,7 +584,7 @@ async def test_tags_directory_partial_file_deleted(test_session, test_directory,
@pytest.mark.asyncio
async def test_tags_directory_all_files_deleted(test_session, test_directory, tag_factory, video_bytes,
image_bytes_factory, await_switches):
image_bytes_factory, await_switches):
"""When ALL files in a FileGroup are deleted, hardlinks should be preserved."""
from wrolpi import tags as tags_module
@ -728,8 +728,6 @@ async def test_recent_tags_api(async_client, test_session, make_files_structure,
assert 'alpha' in body['tag_names']
@pytest.mark.asyncio
async def test_get_overlapping_tags_multiple(test_session, make_files_structure, tag_factory, video_bytes,
refresh_files):

View file

@ -7,7 +7,6 @@ import pathlib
import pytest
from wrolpi.collections.models import Collection
from wrolpi.common import get_media_directory
from wrolpi.media_path import MediaPathType