From ae9d188391da995c01d7b83363198589d1598cba Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 20 Oct 2023 18:53:26 -0600 Subject: [PATCH] Adding FileGroup.published_date/author/etc. Creating script which converts a Singlefile to an Archive FileGroup files. Parsing SingleFile datetime using dates.strpdate. Adding timezone when no timezone info is available. Installing python-dateutil to assist in handling diverse datetime strings. Reworking refresh progress to use idempotency to display more accurate progress. Only logging that files have been invalidated if they used to be indexed. Clearing archives to force re-index. Ignoring 'Unknown' author from PDF metadata. Using file_title when pdf metadata title is empty. --- .gitmodules | 2 +- .../versions/4b54ad052e2e_reindex_archives.py | 32 +++ ...6b0b249a4f81_archive_published_datetime.py | 42 ++++ app/src/components/Archive.js | 29 ++- app/src/components/Common.js | 5 +- app/src/components/Files.js | 6 +- .../config/package-lists/wrolpi.list.chroot | 4 + etc/raspberrypios/50x.html | 92 ++++----- modules/archive/__init__.py | 14 +- modules/archive/conftest.py | 3 +- modules/archive/lib.py | 188 +++++++++++++++--- modules/archive/models.py | 33 ++- modules/archive/test/test_lib.py | 174 +++++++++++++++- modules/inventory/common.py | 30 ++- modules/inventory/inventory.py | 33 ++- modules/inventory/test/test_inventory.py | 19 +- modules/videos/lib.py | 5 +- modules/videos/test/test_downloader.py | 1 - modules/videos/test/test_lib.py | 8 +- modules/videos/video/test/test_api.py | 1 - modules/zim/downloader.py | 6 +- pi-gen/stage2/04-wrolpi/01-packages | 4 + pi-gen/stage2/04-wrolpi/04-run-chroot.sh | 2 +- requirements.txt | 3 +- scripts/singlefile_to_archive.py | 106 ++++++++++ wrolpi/cmd.py | 8 +- wrolpi/common.py | 69 +++---- wrolpi/dates.py | 89 ++++++++- wrolpi/files/downloader.py | 1 - wrolpi/files/ebooks.py | 5 +- wrolpi/files/lib.py | 98 +++++++-- wrolpi/files/models.py | 16 +- wrolpi/files/pdfs.py | 89 ++++++--- wrolpi/files/test/test_lib.py | 2 +- wrolpi/files/test/test_pdfs.py | 10 +- wrolpi/root_api.py | 6 +- wrolpi/scrape_downloader.py | 6 +- wrolpi/test/test_common.py | 22 -- wrolpi/test/test_dates.py | 63 ++++-- wrolpi/vars.py | 2 - 40 files changed, 1023 insertions(+), 305 deletions(-) create mode 100644 alembic/versions/4b54ad052e2e_reindex_archives.py create mode 100644 alembic/versions/6b0b249a4f81_archive_published_datetime.py create mode 100755 scripts/singlefile_to_archive.py diff --git a/.gitmodules b/.gitmodules index 3044883c..7580a762 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "docker/help/wrolpi-help"] path = docker/help/wrolpi-help - url = https://github.com/lrnselfreliance/wrolpi-help.git + url = https://github.com/wrolpi/wrolpi-help.git diff --git a/alembic/versions/4b54ad052e2e_reindex_archives.py b/alembic/versions/4b54ad052e2e_reindex_archives.py new file mode 100644 index 00000000..e84d808c --- /dev/null +++ b/alembic/versions/4b54ad052e2e_reindex_archives.py @@ -0,0 +1,32 @@ +"""Reindex archives. + +Revision ID: 4b54ad052e2e +Revises: 6b0b249a4f81 +Create Date: 2023-10-20 13:28:55.517086 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '4b54ad052e2e' +down_revision = '6b0b249a4f81' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + # Archives must be re-indexed because of the previous revision. + session.execute("UPDATE file_group SET indexed=false WHERE model='archive'") + session.execute("DELETE FROM archive") # noqa + + +def downgrade(): + pass diff --git a/alembic/versions/6b0b249a4f81_archive_published_datetime.py b/alembic/versions/6b0b249a4f81_archive_published_datetime.py new file mode 100644 index 00000000..b3cddf7b --- /dev/null +++ b/alembic/versions/6b0b249a4f81_archive_published_datetime.py @@ -0,0 +1,42 @@ +"""Archive published_datetime + +Revision ID: 6b0b249a4f81 +Revises: 4c8da5502c8a +Create Date: 2023-10-20 12:54:55.879180 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '6b0b249a4f81' +down_revision = '4c8da5502c8a' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE file_group ADD COLUMN author TEXT') + session.execute('ALTER TABLE file_group ADD COLUMN published_datetime TIMESTAMP WITH TIME ZONE') + session.execute('ALTER TABLE file_group ADD COLUMN published_modified_datetime TIMESTAMP WITH TIME ZONE') + session.execute('CREATE INDEX IF NOT EXISTS file_group_author_idx ON file_group(author)') + session.execute('CREATE INDEX IF NOT EXISTS file_group_published_datetime_idx ON file_group(published_datetime)') + session.execute('CREATE INDEX IF NOT EXISTS file_group_published_modified_datetime_idx' + ' ON file_group(published_modified_datetime)') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('DROP INDEX IF EXISTS file_group_author_idx') + session.execute('DROP INDEX IF EXISTS file_group_published_datetime_idx') + session.execute('DROP INDEX IF EXISTS file_group_published_modified_datetime_idx') + session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS author') + session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_datetime') + session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_modified_datetime') diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 0ef9564e..4aa8a15a 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -6,6 +6,8 @@ import { CardMeta, Container, Dropdown, + GridColumn, + GridRow, Image, Input, PlaceholderHeader, @@ -18,7 +20,8 @@ import { BackButton, CardLink, cardTitleWrapper, - encodeMediaPath, ErrorMessage, + encodeMediaPath, + ErrorMessage, ExternalCardLink, FileIcon, findPosterPath, @@ -165,6 +168,13 @@ function ArchivePage() { await fetchArchive(); } + const archivedDatetimeString = isoDatetimeToString(data.archive_datetime, true); + const publishedDatetimeString = archiveFile.published_datetime ? isoDatetimeToString(archiveFile.published_datetime, true) + : 'unknown'; + const modifiedDatetimeString = archiveFile.published_modified_datetime + ? isoDatetimeToString(archiveFile.published_modified_datetime, true) + : 'unknown'; + const aboutPane = { menuItem: 'About', render: () =>
Domain
@@ -175,6 +185,9 @@ function ArchivePage() {
URL

{data.url ? {data.url} : 'N/A'}

+ +
Modified Date
+

{modifiedDatetimeString}

 }; @@ -221,7 +234,19 @@ function ArchivePage() {
{textEllipsis(archiveFile.title || data.url)}
 -
{isoDatetimeToString(data.archive_datetime, true)}
+ +
Author: {archiveFile.author ? archiveFile.author : 'unknown'}
+ + + + +
Published: {publishedDatetimeString}
+ + +
Archived: {archivedDatetimeString}
+ + + {singlefileButton} {updateButton} diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 3baedb9d..4189411f 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -179,7 +179,10 @@ export function isoDatetimeToString(dt, time = false) { let d = ; if (dt && time) { d = new Date(dt); - d = `${d.toDateString()} ${d.getHours()}:${d.getMinutes()}:${d.getSeconds()}`; + const hours = String(d.getHours()).padStart(2, '0'); + const minutes = String(d.getMinutes()).padStart(2, '0'); + const seconds = String(d.getSeconds()).padStart(2, '0'); + d = `${d.toDateString()} ${hours}:${minutes}:${seconds}`; } else if (dt) { d = new Date(dt); d = d.toDateString(); diff --git a/app/src/components/Files.js b/app/src/components/Files.js index dbbb52b1..003a42c0 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -137,11 +137,7 @@ function FileCard({file}) { return ; } - const {data} = file; - let author; - if (data) { - author = data['author']; - } + const author = file.author; const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`; const color = mimetypeColor(file.mimetype); const size = file.size !== null && file.size !== undefined ? humanFileSize(file.size) : null; diff --git a/debian-live-config/config/package-lists/wrolpi.list.chroot b/debian-live-config/config/package-lists/wrolpi.list.chroot index 5629e395..d016cad5 100644 --- a/debian-live-config/config/package-lists/wrolpi.list.chroot +++ b/debian-live-config/config/package-lists/wrolpi.list.chroot @@ -1,5 +1,6 @@ apache2 apache2-dev +apache2-doc apt-transport-https autoconf build-essential @@ -17,8 +18,10 @@ fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted gcc +gcc-doc gdal-bin git +git-doc gnupg-agent hostapd htop @@ -64,6 +67,7 @@ osm2pgsql osmium-tool pkg-config postgis +postgis-doc postgresql-15 postgresql-15-postgis-3 postgresql-15-postgis-3-scripts diff --git a/etc/raspberrypios/50x.html b/etc/raspberrypios/50x.html index 529a1511..5f9dd621 100644 --- a/etc/raspberrypios/50x.html +++ b/etc/raspberrypios/50x.html @@ -1,3 +1,4 @@ + @@ -5,8 +6,8 @@

nginx could not find WROLPi! Are your services running?

-

sudo systemctl status wrolpi-app

-

sudo systemctl status wrolpi-api

+
sudo systemctl status wrolpi-app
+
sudo systemctl status wrolpi-api

If either of those services are not running, start all WROLPi services:

@@ -15,71 +16,72 @@

Or, start them individually:

-

sudo systemctl start wrolpi-app

-

sudo systemctl start wrolpi-api

-

sudo systemctl start wrolpi-kiwix

+
sudo systemctl start wrolpi-app
+
sudo systemctl start wrolpi-api
+
sudo systemctl start wrolpi-kiwix

-

View the logs

-

journalctl -u wrolpi-app

-

journalctl -u wrolpi-api

-

journalctl -u wrolpi-kiwix

-

Watch the logs

-

journalctl -fu wrolpi-app

-

journalctl -fu wrolpi-api

-

journalctl -fu wrolpi-kiwix

+

View the logs:

+
journalctl -u wrolpi-app
+
journalctl -u wrolpi-api
+
journalctl -u wrolpi-kiwix
+

Watch the logs:

+
journalctl -fu wrolpi-app
+
journalctl -fu wrolpi-api
+
journalctl -fu wrolpi-kiwix

Help script

Run the provided help script which may narrow down the scope of the problem.

-

bash /opt/wrolpi/help.sh

+
bash /opt/wrolpi/help.sh

Repair script

Run the provided repair script which will attempt to automatically repair the WROLPi systems.

-

bash /opt/wrolpi/repair.sh

+
bash /opt/wrolpi/repair.sh

Still not working?

You can view the media directory by navigating to /media

+

The API may be available by navigating to port 8081

+

The map may be available by navigating to port 8084

+

Kiwix may be available by navigating to port 8085

-

The API may be available by navigating to port 8081

+
-

The map may be available by navigating to port 8084

- -

Kiwix may be available by navigating to port 8085

+

Third-Party Documentation

+

WROLPi comes with official documentation from some third-party applications:

+ diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 3a80c8b5..3f33e3a5 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -238,20 +238,21 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive: return archive except Exception as e: - logger.error(f'Failed to model Archive {file_group}', exc_info=e) - if PYTEST: - raise + logger.error('Failed to model Archive', exc_info=e) + raise InvalidArchive(f'Failed to model Archive {file_group}') from e @register_modeler async def archive_modeler(): + """Searches DB for FileGroups that contain an HTML file. If the HTML file is a SingleFile, we model it as an + Archive.""" invalid_archives = set() while True: with get_db_session(commit=True) as session: results = session.query(FileGroup, Archive) \ .filter( - # Get all groups that contain a PDF that have not been indexed. + # Get all groups that contain an HTML file that have not been indexed. FileGroup.indexed != True, FileGroup.mimetype == 'text/html', ).filter(not_(FileGroup.id.in_(list(invalid_archives)))) \ @@ -262,7 +263,8 @@ async def archive_modeler(): for file_group, archive in results: processed += 1 - with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}', logger__=logger): + with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}', + logger__=logger): if archive: archive: Archive try: @@ -276,7 +278,7 @@ async def archive_modeler(): try: model_archive(file_group, session=session) except InvalidArchive: - # May not be a real Singlefile archive. + # It was not a real Archive. Many HTML files will not be an Archive. pass # Even if indexing fails, we mark it as indexed. We won't retry indexing this. diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py index 5c82e08e..d8965646 100644 --- a/modules/archive/conftest.py +++ b/modules/archive/conftest.py @@ -76,7 +76,8 @@ def archive_factory(test_session, archive_directory, make_files_structure): archive = Archive.from_paths(test_session, *files) archive.url = url archive.title = title - archive.archive_datetime = next(now) + archive.archive_datetime = archive.file_group.published_datetime = next(now) + archive.file_group.modification_datetime = next(now) archive.domain = domain archive.validate() diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 04f26d11..74c4b246 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -5,19 +5,22 @@ import pathlib import re from dataclasses import dataclass from datetime import datetime -from typing import Optional, Tuple, List +from typing import Optional, Tuple, List, Union +from urllib.parse import urlparse -from bs4 import BeautifulSoup +import pytz from sqlalchemy import asc from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive +from wrolpi import dates from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \ - format_html_string, split_lines_by_length + format_html_string, split_lines_by_length, get_html_soup from wrolpi.dates import now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import UnknownArchive, InvalidOrderBy from wrolpi.tags import tag_names_to_file_group_sub_select +from wrolpi.vars import PYTEST logger = logger.getChild(__name__) @@ -220,13 +223,157 @@ def get_title_from_html(html: str, url: str = None) -> str: """ Try and get the title from the """ - soup = BeautifulSoup(html, features='html.parser') + soup = get_html_soup(html) try: return soup.title.string except Exception: # noqa logger.debug(f'Unable to extract title {url}') +SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n') + + +def get_url_from_singlefile(html: bytes) -> str: + """Extract URL from SingleFile contents.""" + html = html.decode() + if SINGLEFILE_HEADER not in html: + raise RuntimeError(f'Not a singlefile!') + + url, = SINGLEFILE_URL_EXTRACTOR.findall(html) + url = url.strip() + + if not url.startswith('http'): + raise RuntimeError('URL did not start with http') + + # Verify that the URL can be parsed. + urlparse(url) + return url + + +@dataclass +class ArticleMetadata: + title: str = None + published_datetime: datetime = None + modified_datetime: datetime = None + description: str = None + author: str = None + + +def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True) -> ArticleMetadata: + """ + Read the data from the tags, extract any data relevant to the article. This function also reads the + +''' singlefile, *_ = make_files_structure({ 'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents, @@ -543,8 +553,14 @@ async def test_refresh_archives_index(test_session, make_files_structure): archive: Archive = test_session.query(Archive).one() assert archive.singlefile_path == singlefile - assert archive.file_group.a_text == 'the title' - assert archive.file_group.d_text == 'article text contents' + + assert archive.file_group.author == 'AUTHOR NAME' + assert archive.file_group.published_datetime + assert archive.file_group.published_modified_datetime + + assert archive.file_group.a_text == 'the title' # from readability json + assert archive.file_group.b_text == 'The article description' # From application/ld+json + assert archive.file_group.d_text == 'article text contents' # from readability txt assert archive.file_group.indexed is True @@ -651,3 +667,143 @@ def test_archive_history(test_session, archive_factory): assert not archive3.history, 'archive3 has no history' assert not archive4.history, 'archive4 has no history' assert not archive5.history, 'archive5 has no history' + + +SINGLEFILE_1 = b''' + + + +''' + +SINGLEFILE_2 = b''' + + + + + This is not a meta title and should be ignored + + + + + + + + +''' + +SINGLEFILE_3 = b''' + + +This is not a meta title and should be ignored + +''' + +SINGLEFILE_4 = b''' + + +This is not a meta title and should be ignored + +''' + +SINGLEFILE_5 = b''' + + +This is not a meta title and should be ignored + + + +''' + + +def test_parse_article_html_metadata(): + """Singlefiles may contain metadata tags.""" + # Metadata can be empty. + metadata = lib.parse_article_html_metadata(SINGLEFILE_1) + assert metadata.title is None + assert metadata.published_datetime is None + assert metadata.modified_datetime is None + assert metadata.description is None + assert metadata.author is None + + # Data from elements. + metadata = lib.parse_article_html_metadata(SINGLEFILE_2) + assert metadata.title == 'The Title' + assert metadata.published_datetime == datetime(2023, 10, 18, 4, 52, 23, tzinfo=utc) + assert metadata.modified_datetime == datetime(2023, 10, 19, 5, 53, 24, tzinfo=utc) + assert metadata.description is None + assert metadata.author == 'Author Name' + + # Data from Article structured data + # https://developers.google.com/search/docs/appearance/structured-data/article + metadata = lib.parse_article_html_metadata(SINGLEFILE_3) + assert metadata.title == 'The headline' + assert metadata.published_datetime == datetime(2022, 9, 27, 0, 40, 19, tzinfo=utc) + assert metadata.modified_datetime == datetime(2022, 9, 27, 13, 43, 47, 971000, tzinfo=utc) + assert metadata.description == 'The article description' + assert metadata.author == 'BOBBY' + + # Data from author anchor element. + metadata = lib.parse_article_html_metadata(SINGLEFILE_4) + assert metadata.author == 'Link Author' + + # author anchor may be a list. + metadata = lib.parse_article_html_metadata(SINGLEFILE_5) + assert metadata.author == 'The Span Author' + assert metadata.published_datetime == datetime(2023, 8, 25, tzinfo=pytz.UTC) + + +@pytest.mark.parametrize( + 'html,expected', + [ + ('', + datetime(2023, 10, 18, 4, 52, 23, tzinfo=utc)), + ('', + datetime(2022, 3, 17, 10, 0, tzinfo=pytz.UTC)), + ('', + datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)), + ('', + datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)), + ( + '', + datetime(2013, 3, 5, 6, 12, tzinfo=pytz.UTC)), + ('', + datetime(2023, 8, 25, tzinfo=pytz.UTC)) + ]) +def test_published_datetime(html, expected): + assert lib.parse_article_html_metadata(html).published_datetime == expected + + +@pytest.mark.parametrize( + 'html,expected', + [ + ('', + datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)), + ('', + datetime(2023, 10, 19, 5, 53, 24, tzinfo=utc)), + ( + '', + datetime(2022, 9, 27, 13, 43, 47, 971000, tzinfo=utc)), + ]) +def test_modified_datetime(html, expected): + assert lib.parse_article_html_metadata(html).modified_datetime == expected + + +@pytest.mark.parametrize( + 'html,expected', + [ + ('', 'Linda'), + ('', 'Billy'), + ] +) +def test_author(html, expected): + assert lib.parse_article_html_metadata(html).author == expected diff --git a/modules/inventory/common.py b/modules/inventory/common.py index 2990a6e5..a8020b8a 100644 --- a/modules/inventory/common.py +++ b/modules/inventory/common.py @@ -9,8 +9,8 @@ from wrolpi import before_startup from wrolpi.common import logger, Base, ConfigFile from wrolpi.db import get_db_session from .errors import NoInventories, InventoriesVersionMismatch -from .inventory import unit_registry, get_items, get_inventories, increment_inventories_version, \ - get_inventories_version +from .inventory import get_items, get_inventories, increment_inventories_version, \ + get_inventories_version, get_unit_registry from .models import Inventory, Item MY_DIR: Path = Path(__file__).parent @@ -23,6 +23,8 @@ def sum_by_key(items: List[Base], key: callable): Sum the total size of each item by the provided key function. Returns a dict containing the key, and the total_size for that key. Combine total of like units. This means ounces and pounds will be in the same total. """ + unit_registry = get_unit_registry() + summed = dict() for item in items: k = key(item) @@ -61,17 +63,28 @@ def get_inventory_by_keys(keys: Tuple, inventory_id: int): get_inventory_by_category = partial(get_inventory_by_keys, ('category',)) get_inventory_by_subcategory = partial(get_inventory_by_keys, ('category', 'subcategory')) get_inventory_by_name = partial(get_inventory_by_keys, ('brand', 'name')) -INVENTORY_UNITS = { - ('ounce', 1): (16, unit_registry.pound), - ('pound', 1): (2000, unit_registry.ton), -} +INVENTORY_UNITS: dict = None UNIT_PRECISION = 5 -def compact_unit(quantity: unit_registry.Quantity) -> unit_registry.Quantity: +def initialize_inventory_units() -> dict: + global INVENTORY_UNITS + if INVENTORY_UNITS: + return INVENTORY_UNITS + + unit_registry = get_unit_registry() + INVENTORY_UNITS = { + ('ounce', 1): (16, unit_registry.pound), + ('pound', 1): (2000, unit_registry.ton), + } + + +def compact_unit(quantity: Quantity) -> Quantity: """ Convert a Quantity to it's more readable format. Such as 2000 pounds to 1 ton. """ + initialize_inventory_units() + number, (units,) = quantity.to_tuple() next_unit = INVENTORY_UNITS.get(units) if not next_unit: @@ -88,7 +101,8 @@ def compact_unit(quantity: unit_registry.Quantity) -> unit_registry.Quantity: return quantity -def quantity_to_tuple(quantity: unit_registry.Quantity) -> Tuple[Decimal, Quantity]: +def quantity_to_tuple(quantity: Quantity) -> Tuple[Decimal, Quantity]: + unit_registry = get_unit_registry() decimal, (units,) = quantity.to_tuple() unit, _ = units quantity, unit = round(decimal, UNIT_PRECISION), unit_registry(unit) diff --git a/modules/inventory/inventory.py b/modules/inventory/inventory.py index 6705dd6b..91208440 100644 --- a/modules/inventory/inventory.py +++ b/modules/inventory/inventory.py @@ -15,7 +15,38 @@ from .models import Inventory, Item, InventoriesVersion logger = logger.getChild(__name__) -unit_registry = pint.UnitRegistry() +__all__ = [ + 'DEFAULT_CATEGORIES', + 'DEFAULT_INVENTORIES', + 'delete_inventory', + 'delete_items', + 'get_brands', + 'get_categories', + 'get_inventories', + 'get_inventories_version', + 'get_items', + 'get_next_inventories_version', + 'get_unit_registry', + 'increment_inventories_version', + 'logger', + 'save_inventory', + 'save_item', + 'sort_categories', + 'update_inventory', + 'update_item', +] + +UNIT_REGISTRY: pint.UnitRegistry = None + + +def get_unit_registry() -> pint.UnitRegistry: + global UNIT_REGISTRY + if UNIT_REGISTRY: + return UNIT_REGISTRY + + UNIT_REGISTRY = pint.UnitRegistry() + return UNIT_REGISTRY + DEFAULT_CATEGORIES = [ ('salt', 'cooking ingredients'), diff --git a/modules/inventory/test/test_inventory.py b/modules/inventory/test/test_inventory.py index 8a234073..9d2e9565 100644 --- a/modules/inventory/test/test_inventory.py +++ b/modules/inventory/test/test_inventory.py @@ -15,11 +15,12 @@ from .. import init from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \ compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file, get_inventories_config from ..errors import NoInventories, InventoriesVersionMismatch -from ..inventory import unit_registry, \ - get_inventories, save_inventory, update_inventory, \ - delete_inventory, get_categories +from ..inventory import get_inventories, save_inventory, update_inventory, \ + delete_inventory, get_categories, get_unit_registry from ..models import Item, Inventory +UNIT_REGISTRY = get_unit_registry() + TEST_ITEMS_COLUMNS = ( 'inventory_id', 'brand', @@ -199,11 +200,11 @@ def quantity_to_string(quantity: Quantity) -> str: @pytest.mark.parametrize( 'quantity,expected', [ - (Decimal(5) * unit_registry.ounce, '5 ounce'), - (Decimal(16) * unit_registry.ounce, '1 pound'), - (Decimal(500) * unit_registry.pound, '500 pound'), - (Decimal(2000) * unit_registry.pound, '1 ton'), - (Decimal(128000) * unit_registry.ounce, '4 ton'), + (Decimal(5) * UNIT_REGISTRY.ounce, '5 ounce'), + (Decimal(16) * UNIT_REGISTRY.ounce, '1 pound'), + (Decimal(500) * UNIT_REGISTRY.pound, '500 pound'), + (Decimal(2000) * UNIT_REGISTRY.pound, '1 ton'), + (Decimal(128000) * UNIT_REGISTRY.ounce, '4 ton'), ] ) def test_compact_unit(quantity, expected): @@ -211,7 +212,7 @@ def test_compact_unit(quantity, expected): assert quantity_to_string(compact_unit(quantity)) == expected -pound, oz, gram, gallon = unit_registry('pound'), unit_registry('oz'), unit_registry('gram'), unit_registry('gallon') +pound, oz, gram, gallon = UNIT_REGISTRY('pound'), UNIT_REGISTRY('oz'), UNIT_REGISTRY('gram'), UNIT_REGISTRY('gallon') pound, oz, gram, gallon = pound.units, oz.units, gram.units, gallon.units mass = pound.dimensionality length = gallon.dimensionality diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 3530b7d7..56671428 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -6,6 +6,7 @@ import re import warnings from typing import Tuple, Optional, Generator +import pytz from sqlalchemy import or_ from sqlalchemy.orm import Session from yt_dlp import YoutubeDL @@ -102,6 +103,8 @@ def validate_video(video: Video, channel_generate_poster: bool): upload_date = upload_date or video.upload_date if upload_date and isinstance(upload_date, str): upload_date = dates.strpdate(upload_date) + if not upload_date.tzinfo: + upload_date = upload_date.astimezone(pytz.UTC) video.file_group.title = video.file_group.title or html.unescape(title) video.upload_date = upload_date video.source_id = video.source_id or source_id @@ -120,7 +123,7 @@ def validate_video(video: Video, channel_generate_poster: bool): if video.ffprobe_json: if duration := video.ffprobe_json['format'].get('duration'): video.duration = float(duration) - elif video_streams := video.get_streams_by_codec_type('video'): + elif (video_streams := video.get_streams_by_codec_type('video')) and 'duration' in video_streams[0]: video.duration = float(video_streams[0]['duration']) else: # Slowest method. diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index a4360cb0..1e6f3c21 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -4,7 +4,6 @@ from copy import copy from unittest import mock import pytest -from yt_dlp.utils import UnsupportedError from modules.videos.channel.lib import download_channel from modules.videos.downloader import VideoDownloader, \ diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 5248f08a..a786358b 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -1,4 +1,3 @@ -import datetime import pathlib import shutil @@ -7,10 +6,8 @@ import pytest from modules.videos import lib from modules.videos.lib import parse_video_file_name, validate_video, get_statistics from modules.videos.models import Video -from wrolpi.dates import now from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup -from wrolpi.tags import Tag from wrolpi.vars import PROJECT_DIR @@ -67,7 +64,7 @@ async def test_video_factory(test_session, video_factory, channel_factory): assert video.source_id == 'some id' -def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file): +def test_validate_video(test_directory, video_factory, image_bytes_factory, video_file): """A video poster will be generated only if the channel permits.""" video_file = video_file.rename(test_directory / 'Channel Name_20050607_1234567890_The Title.mp4') vid1 = video_factory(with_video_file=video_file) @@ -81,7 +78,8 @@ def test_validate_video(test_session, test_directory, video_factory, image_bytes assert vid1.poster_path.is_file(), 'Poster path does not exist' assert vid1.video_path.stem == vid1.poster_path.stem assert vid1.poster_path.suffix == '.jpg' - assert vid1.upload_date == datetime.datetime(2005, 6, 7, 0, 0, tzinfo=datetime.timezone.utc) + # File name date is assumed to be local timezone. + assert vid1.upload_date and vid1.upload_date.year == 2005 assert vid1.source_id == '1234567890' assert vid1.file_group.title == 'The Title' diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py index 4d6d7ad5..1619f466 100644 --- a/modules/videos/video/test/test_api.py +++ b/modules/videos/video/test/test_api.py @@ -6,7 +6,6 @@ from json import dumps import pytest from modules.videos.models import Video -from modules.videos.video.lib import get_video_for_app from wrolpi.dates import now diff --git a/modules/zim/downloader.py b/modules/zim/downloader.py index d75b9a45..86f77839 100644 --- a/modules/zim/downloader.py +++ b/modules/zim/downloader.py @@ -1,10 +1,8 @@ from http import HTTPStatus from typing import List -import bs4 - from modules.zim import lib -from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task +from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task, get_html_soup from wrolpi.downloader import Downloader, Download, DownloadResult __all__ = ['KiwixCatalogDownloader', 'KiwixZimDownloader', 'kiwix_zim_downloader', 'kiwix_catalog_downloader'] @@ -20,7 +18,7 @@ async def fetch_hrefs(url: str) -> List[str]: if status != HTTPStatus.OK: raise RuntimeError(f'Failed to fetch file catalog') - soup = bs4.BeautifulSoup(content, 'html.parser') + soup = get_html_soup(content) downloads = list() for a_ in soup.find_all('a'): downloads.append(a_['href']) diff --git a/pi-gen/stage2/04-wrolpi/01-packages b/pi-gen/stage2/04-wrolpi/01-packages index 3372d805..d35e5d4e 100644 --- a/pi-gen/stage2/04-wrolpi/01-packages +++ b/pi-gen/stage2/04-wrolpi/01-packages @@ -1,5 +1,6 @@ apache2 apache2-dev +apache2-doc apt-transport-https autoconf build-essential @@ -17,8 +18,10 @@ fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted gcc +gcc-doc gdal-bin git +git-doc gnupg-agent hostapd iotop @@ -62,6 +65,7 @@ osm2pgsql osmium-tool pkg-config postgis +postgis-doc postgresql-15 postgresql-15-postgis-3 postgresql-15-postgis-3-scripts diff --git a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh index 8a9d7b5c..1a4442dc 100755 --- a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh @@ -43,7 +43,7 @@ chmod 644 /var/www/html/* # Allow wrolpi user to delete map tiles. usermod -aG _renderd wrolpi -mkdir /var/cache/renderd/tiles +[ ! -d /var/cache/renderd/tiles ] && mkdir /var/cache/renderd/tiles chown -R _renderd:_renderd /var/cache/renderd/tiles # Create the media directory for the wrolpi user. diff --git a/requirements.txt b/requirements.txt index f1fe90af..3280531a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ Pillow==10.0.1 Pint==0.21 -pypdf==3.12.1 PyYAML==6.0 SQLAlchemy[asyncio]==1.3.22 aiohttp==3.8.5 @@ -13,8 +12,10 @@ libzim==3.1.0 mock==5.0.2 psutil==5.9.5 psycopg2-binary==2.9.6 +pypdf==3.12.1 pytest-asyncio==0.21.0 pytest==7.3.1 +python-dateutil==2.8.2 python-dotenv==1.0.0 python-magic~=0.4.27 pytz==2023.3 diff --git a/scripts/singlefile_to_archive.py b/scripts/singlefile_to_archive.py new file mode 100755 index 00000000..894c03f2 --- /dev/null +++ b/scripts/singlefile_to_archive.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Extracts Readability, screenshot, etc. from a Singlefile.""" +import argparse +import json +import logging +import pathlib +import subprocess +import sys +import tempfile + +from modules.archive import lib as archive_lib +from wrolpi.cmd import READABILITY_BIN +from wrolpi.common import html_screenshot, format_html_string, logger + +logger = logger.getChild('singlefile_to_archive') + + +def do_readability(html: bytes, url: str) -> dict: + """Extract the readability dict from the provided HTML.""" + with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh: + fh.write(html) + + cmd = (READABILITY_BIN, fh.name, url) + output = subprocess.check_output(cmd, timeout=60) + + readability = json.loads(output) + return readability + + +def singlefile_to_archive_files(singlefile: pathlib.Path): + """Extract Readability files and Screenshot file next to the provided Singlefile.""" + contents = singlefile.read_bytes() + + readability_html_path = singlefile.with_suffix('.readability.html') + readability_json_path = singlefile.with_suffix('.readability.json') + readability_text_path = singlefile.with_suffix('.readability.txt') + + # Only extract Readability if any of the files are missing. + readability = dict() + if not readability_html_path.is_file() or not readability_text_path.is_file() or not readability_json_path.is_file(): + url = archive_lib.get_url_from_singlefile(contents) + try: + readability = do_readability(contents, url) + except Exception as e: + logger.error(f'Failed to run readability', exc_info=e) + + # The real Readability the user wants. + readability_html = readability.pop('content', None) + if readability and not readability_html_path.is_file() and readability_html: + try: + readability_html = format_html_string(readability_html) + readability_html_path.write_text(readability_html) + except Exception as e: + logger.error(f'Failed to extract readability: {singlefile}', exc_info=e) + + # The text from within the Readability, used for searching. + readability_text = readability.pop('textContent', None) + if readability and not readability_text_path.is_file() and readability_text: + readability_text_path.write_text(readability_text) + + # Write the JSON last now that content/textContent have been removed. + if readability and not readability_json_path.is_file(): + try: + with readability_json_path.open('wt') as fh: + fh.write(json.dumps(readability, indent=2)) + except Exception as e: + logger.error(f'Failed to extract Readability text: {singlefile}', exc_info=e) + + # Only generate a screenshot if the file is missing. + screenshot_path = singlefile.with_suffix('.png') + if not screenshot_path.is_file(): + try: + screenshot = html_screenshot(contents) + screenshot_path.write_bytes(screenshot) + except Exception as e: + logger.error(f'Failed to create screenshot: {singlefile}', exc_info=e) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('files', nargs='+') + parser.add_argument('-v', action='count') + args = parser.parse_args() + + if args.v == 0: + logger.setLevel(logging.WARNING) + elif args.v == 1: + logger.setLevel(logging.INFO) + elif args.v >= 2: + logger.setLevel(logging.DEBUG) + logger.debug(f'Debug logging') + + if not READABILITY_BIN: + print(f'Cannot find "readability-extractor". Install it.') + sys.exit(1) + + files = [pathlib.Path(i) for i in args.files] + invalid_files = [i for i in files if not archive_lib.is_singlefile_file(i)] + if invalid_files: + for i in invalid_files: + print(f'File is not a singlefile: {i}') + sys.exit(1) + + for file in files: + singlefile_to_archive_files(file) + print(f'Finished {file}') diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py index 08b75a3f..382ab957 100644 --- a/wrolpi/cmd.py +++ b/wrolpi/cmd.py @@ -55,25 +55,22 @@ CPUFREQ_SET_BIN = which('cpufreq-set', '/usr/bin/cpufreq-set') WGET_BIN = which('wget', '/usr/bin/wget', flag=flags.wget_installed) # Map -BASH_BIN = which('bash', '/bin/bash', warn=True) +BASH_BIN = which('bash', '/bin/bash') # Archive SINGLE_FILE_BIN = which('single-file', '/usr/bin/single-file', # rpi os '/usr/local/bin/single-file', # debian - warn=True, flag=flags.singlefile_installed, ) CHROMIUM = which('chromium-browser', 'chromium', '/usr/bin/chromium-browser', # rpi os '/usr/bin/chromium', # debian - warn=True, flag=flags.chromium_installed, ) READABILITY_BIN = which('readability-extractor', '/usr/bin/readability-extractor', # rpi os '/usr/local/bin/readability-extractor', # debian - warn=True, flag=flags.readability_installed, ) @@ -82,8 +79,7 @@ YT_DLP_BIN = which( 'yt-dlp', '/usr/local/bin/yt-dlp', # Location in docker container '/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location - warn=True, flag=flags.yt_dlp_installed, ) -FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe', warn=True, flag=flags.ffprobe_installed) +FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe', flag=flags.ffprobe_installed) FFMPEG_BIN = which('ffmpeg', '/usr/bin/ffmpeg', flag=flags.ffmpeg_installed) diff --git a/wrolpi/common.py b/wrolpi/common.py index ab231200..1545bde6 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -27,6 +27,7 @@ from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Gener from urllib.parse import urlparse, urlunsplit import aiohttp +import bs4 import yaml from aiohttp import ClientResponse from bs4 import BeautifulSoup @@ -36,7 +37,6 @@ from sqlalchemy.ext.compiler import compiles from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Session -from wrolpi import dates from wrolpi.dates import now, from_timestamp, seconds_to_timestamp from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS @@ -129,8 +129,8 @@ __all__ = [ 'get_config', 'get_download_info', 'get_files_and_directories', + 'get_html_soup', 'get_media_directory', - 'get_model_by_table_name', 'get_relative_to_media_directory', 'get_warn_once', 'insert_parameter', @@ -138,7 +138,6 @@ __all__ = [ 'limit_concurrent', 'logger', 'make_media_directory', - 'match_paths_to_suffixes', 'minimize_dict', 'native_only', 'partition', @@ -147,7 +146,6 @@ __all__ = [ 'register_refresh_cleanup', 'remove_whitespace', 'run_after', - 'sanitize_link', 'set_log_level', 'set_test_config', 'set_test_media_directory', @@ -161,6 +159,8 @@ __all__ = [ 'wrol_mode_check', 'wrol_mode_enabled', 'zig_zag', + 'html_screenshot', + 'format_html_string', ] # Base is used for all SQLAlchemy models. @@ -200,13 +200,6 @@ class ModelHelper: Session.object_session(self).flush([self]) -def get_model_by_table_name(table_name): - """Find a model by its table name.""" - for klass in Base._decl_class_registry.values(): - if hasattr(klass, '__tablename__') and klass.__tablename__ == table_name: - return klass - - class tsvector(types.TypeDecorator): impl = types.UnicodeText @@ -219,12 +212,6 @@ def compile_tsvector(element, compiler, **kw): URL_CHARS = string.ascii_lowercase + string.digits -def sanitize_link(link: str) -> str: - """Remove any non-url safe characters, all will be lowercase.""" - new_link = ''.join(i for i in str(link).lower() if i in URL_CHARS) - return new_link - - class ConfigFile: """ This class keeps track of the contents of a config file. You can update the config (and it's file) by @@ -1118,37 +1105,21 @@ def register_refresh_cleanup(func: callable): async def apply_refresh_cleanup(): + # TODO convert all functions to async so refresh functions can be used. for func in REFRESH_CLEANUP: - start = dates.now() - try: - logger_.info(f'Applying refresh cleanup {func.__name__}') - func() - except Exception as e: - logger_.error(f'Refresh cleanup {func.__name__} failed!', exc_info=e) - if PYTEST: - raise - finally: - elapsed = (dates.now() - start).total_seconds() - logger_.warning(f'After refresh cleanup {func.__name__} took {int(elapsed)} seconds') + slow_message = f'After refresh cleanup {func.__name__} took %(elapsed)s seconds' + with slow_logger(5, slow_message, logger__=logger_): + try: + logger_.info(f'Applying refresh cleanup {func.__name__}') + func() + except Exception as e: + logger_.error(f'Refresh cleanup {func.__name__} failed!', exc_info=e) + if PYTEST: + raise # Sleep to catch cancel. await asyncio.sleep(0) -@iterify(tuple) -def match_paths_to_suffixes(paths: List[pathlib.Path], suffix_groups: List[Tuple[str]]): - paths = paths.copy() - for group in suffix_groups: - match = None - for suffix in group: - # Compare all suffixes in the group to the path's name. Yield the first path that shares the first - # suffix. If no suffix matches any path, yield None. - match = next(filter(lambda i: i.name.endswith(suffix), paths), None) - if match: - paths.pop(paths.index(match)) - break - yield match - - def chunks(it: Iterable, size: int): """Split an iterable into iterables of the defined length.""" it = iter(it) @@ -1210,7 +1181,7 @@ def timer(name): @contextlib.contextmanager -def slow_logger(max_seconds: int, message: str, logger__=logger, level=logging.WARNING): +def slow_logger(max_seconds: int, message: str, logger__=logger_, level=logging.WARNING): """Only logs when the context duration exceeds the `max_seconds`.""" before = now() try: @@ -1472,8 +1443,13 @@ def url_strip_host(url: str) -> str: return url or '/' -def extract_html_text(html: str) -> str: +def get_html_soup(html: Union[bytes, str]) -> bs4.BeautifulSoup: soup = BeautifulSoup(html, features='html.parser') + return soup + + +def extract_html_text(html: str) -> str: + soup = get_html_soup(html) # kill all script and style elements for script in soup(["script", "style"]): @@ -1538,6 +1514,9 @@ def format_json_file(file: pathlib.Path, indent: int = 2): def format_html_string(html: str) -> str: + if not html.strip(): + raise RuntimeError('Refusing to format empty HTMl string.') + soup = BeautifulSoup(html, features='lxml') return soup.prettify() diff --git a/wrolpi/dates.py b/wrolpi/dates.py index 7daec46c..593091ad 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -1,8 +1,10 @@ import logging +import re from datetime import datetime, timezone, timedelta from enum import Enum from typing import Union +import dateutil.parser import pytz from sqlalchemy import types @@ -41,18 +43,84 @@ def strftime(dt: datetime) -> str: return dt.isoformat() -def strptime(dt: str) -> datetime: - if isinstance(dt, str) and dt.endswith(' 00:00'): - # URL decoding removed + - dt = f'{dt[:26]}+00:00' - return datetime.fromisoformat(dt).astimezone(pytz.UTC) - - def strpdate(dt: str) -> datetime: - if '-' in dt: - return datetime.strptime(dt, '%Y-%m-%d').replace(tzinfo=pytz.UTC) + """ + Attempt to parse a datetime string. Tries to find a Timezone, if possible. DB requires timezone. - return datetime.strptime(dt, '%Y%m%d').replace(tzinfo=pytz.UTC) + TODO replace this in Python 3.11 https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat + """ + try: + if dt.count('/') == 2: + # No timezone info. + try: + a, b, c = dt.split('/') + if len(c) == 4: + # Assume m/d/Y + return datetime.strptime(dt, '%m/%d/%Y') + if len(a) == 4: + # Y/m/d + return datetime.strptime(dt, '%Y/%m/%d') + except ValueError: + pass + elif dt.count('-') == 2 and len(dt) <= 10: + # No timezone info. + a, b, c = dt.split('-') + if len(a) == 4: + # Y-m-d + return datetime.strptime(dt, '%Y-%m-%d') + + if dt.startswith('D:'): + # D:20221226113758-07'00, D:20200205184724, etc. + dt2 = dt.replace("'", '').rstrip('+') + try: + return datetime.strptime(dt2, "D:%Y%m%d%H%M%S%z") + except ValueError: + pass + + return datetime.strptime(dt2, "D:%Y%m%d%H%M%S") + + if dt.count('-') == 0 and dt.count('/') == 0 and dt.isdigit() and len(dt) == 8: + # Assume %Y%m%d date + return datetime.strptime(dt, '%Y%m%d') + + if dt.count('.') == 2 and len(dt) <= 9: + # yyyy.mm.dd or dd.mm.yyyy + try: + a, b, c = dt.split('.') + if len(a) == 4: + return datetime.strptime(dt, '%Y.%m.%d') + if len(c) == 4: + return datetime.strptime(dt, '%d.%m.%Y') + except ValueError: + pass + + try: + # Fri Jun 17 2022 19:24:52 (from Singlefile) + return datetime.strptime(dt, '%a %b %d %Y %H:%M:%S') + except ValueError: + pass + + if len(dt) == 4 and dt.isdigit(): + # Assume this is a year + return datetime.strptime(dt, '%Y').replace(tzinfo=pytz.UTC) + + if ',' in dt and (dt.endswith('AM') or dt.endswith('PM')): + try: + # Tuesday, October 19, 1999 3:41:01 PM + return datetime.strptime(dt, '%A, %B %d, %Y %I:%M:%S %p') + except ValueError: + pass + + try: + # Tue, October 19, 1999 3:41:01 PM + return datetime.strptime(dt, '%a, %B %d, %Y %I:%M:%S %p') + except ValueError: + pass + + # Last, try third-party module to parse ISO 8601 datetime. + return dateutil.parser.isoparse(dt) + except Exception as e: + raise RuntimeError(f'Unable to parse datetime string: {dt}') from e def strftime_ms(dt: datetime) -> str: @@ -73,6 +141,7 @@ def seconds_to_timestamp(seconds: Union[int, float]) -> str: >>> seconds_to_timestamp(5) # '00:00:05' >>> seconds_to_timestamp(60) + # '00:01:00' >>> seconds_to_timestamp(86400) # '1d 00:00:00' diff --git a/wrolpi/files/downloader.py b/wrolpi/files/downloader.py index 8c553e9a..367fd04b 100644 --- a/wrolpi/files/downloader.py +++ b/wrolpi/files/downloader.py @@ -1,4 +1,3 @@ -import asyncio import pathlib import traceback from abc import ABC diff --git a/wrolpi/files/ebooks.py b/wrolpi/files/ebooks.py index dde3591a..0b76b153 100644 --- a/wrolpi/files/ebooks.py +++ b/wrolpi/files/ebooks.py @@ -4,11 +4,10 @@ import logging import pathlib from typing import List, Optional, Tuple -import bs4 from sqlalchemy import Column, Integer, String, BigInteger, ForeignKey, or_, and_ from sqlalchemy.orm import relationship -from wrolpi.common import ModelHelper, Base, register_modeler +from wrolpi.common import ModelHelper, Base, register_modeler, get_html_soup from wrolpi.db import get_db_session from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype from wrolpi.files.models import FileGroup @@ -66,7 +65,7 @@ class EBookData: def extract_text(html: str) -> str: - soup = bs4.BeautifulSoup(html, features='html.parser') + soup = get_html_soup(html) text = soup.get_text() return text diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 04381cdd..ba8762b0 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1,4 +1,5 @@ import asyncio +import dataclasses import datetime import functools import glob @@ -412,14 +413,16 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): AND file_group.size = EXCLUDED.size AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files) ) - RETURNING id, indexed + RETURNING id, file_group.indexed AS old_indexed, indexed ''' curs.execute(stmt) - need_index = [i[0] for i in curs.fetchall() if i[1] is False] - if need_index: - refresh_logger.info(f'Invalidated indexes of {len(need_index)} file groups near {str(chunk[0])}') + results = list(curs.fetchall()) + # Count the files that used to be indexed, but need to be re-indexed. + invalidated_files = len([i['id'] for i in results if i['old_indexed'] and not i['indexed']]) + if invalidated_files: + refresh_logger.info(f'Invalidated indexes of {invalidated_files} file groups near {chunk[0]}') else: - refresh_logger.debug(f'Upserted {len(chunk)} files near {str(chunk[0])}') + refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}') if non_primary_files: # New files may have been added which change what primary paths exist. Delete any file_groups which @@ -456,9 +459,15 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim directories, files = partition(lambda i: i.is_dir(), exists) while directories: directory = directories.pop(0) - new_files, new_directories = get_files_and_directories(directory) + try: + new_files, new_directories = get_files_and_directories(directory) + except FileNotFoundError as e: + # Directory may have been deleted during refresh. + logger.warning(f'Cannot refresh directory because it is missing: {directory}', exc_info=e) + continue directories.extend(new_directories) files.extend(new_files) + refresh_logger.debug(f'Discovered {len(new_files)} files in {directory}') if len(files) >= 100: # Wait until there are enough files to perform the upsert. try: @@ -475,6 +484,8 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim except Exception as e: refresh_logger.error(f'Failed to upsert files', exc_info=e) + refresh_logger.info('Finished discovering files. Will now remove deleted files from DB...') + with get_db_curs(commit=True) as curs: wheres = '' if paths: @@ -512,6 +523,9 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr if isinstance(paths, pathlib.Path): paths = [paths, ] + idempotency = now() + REFRESH['idempotency'] = idempotency + refreshing_all_files = False with flags.refreshing: @@ -523,8 +537,6 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr if send_events: Events.send_global_refresh_started() - idempotency = now() - # Add all files in the media directory to the DB. paths = paths or [get_media_directory()] @@ -591,6 +603,7 @@ async def apply_indexers(): file_groups = session.query(FileGroup).filter(FileGroup.indexed != True).limit(20) file_groups: List[FileGroup] = list(file_groups) + file_group = None processed = 0 for file_group in file_groups: processed += 1 @@ -606,7 +619,8 @@ async def apply_indexers(): # Sleep to catch cancel. await asyncio.sleep(0) - refresh_logger.debug(f'Indexed {processed} files') + if file_group: + refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}') if processed < 20: # Processed less than the limit, don't do the next query. @@ -969,9 +983,52 @@ async def remove_file_group_tag(file_group_id: int, file_group_primary_path: str file_group.remove_tag(tag, session) -def get_refresh_progress(): - with get_db_curs() as curs: - curs.execute(''' +@dataclasses.dataclass +class RefreshProgress: + counted_files: int = 0 + counting: bool = False + discovery: bool = False + indexed: int = 0 + indexing: bool = False + modeled: int = 0 + modeling: bool = False + cleanup: bool = False + refreshing: bool = False + total_file_groups: int = 0 + unindexed: int = 0 + + def __json__(self): + d = dict( + counted_files=self.counted_files, + counting=self.counting, + discovery=self.discovery, + indexed=self.indexed, + indexing=self.indexing, + modeled=self.modeled, + modeling=self.modeling, + cleanup=self.cleanup, + refreshing=self.refreshing, + total_file_groups=self.total_file_groups, + unindexed=self.unindexed, + ) + return d + + +def get_refresh_progress() -> RefreshProgress: + idempotency = REFRESH.get('idempotency') + if idempotency: + stmt = ''' + SELECT + -- Sum all the files in each FileGroup. + SUM(json_array_length(files)) FILTER (WHERE idempotency=%(idempotency)s) AS "total_file_groups", + COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency=%(idempotency)s) AS "indexed", + COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed", + COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency=%(idempotency)s) AS "modeled" + FROM file_group + ''' + else: + # Idempotency has not yet been declared. + stmt = ''' SELECT -- Sum all the files in each FileGroup. SUM(json_array_length(files)) AS "total_file_groups", @@ -979,25 +1036,28 @@ def get_refresh_progress(): COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed", COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled" FROM file_group - ''') + ''' + + with get_db_curs() as curs: + curs.execute(stmt, dict(idempotency=idempotency)) results = dict(curs.fetchone()) # TODO counts are wrong if we are not refreshing all files. - status = dict( + progress = RefreshProgress( counted_files=REFRESH.get('counted_files', 0), counting=flags.refresh_counting.is_set(), discovery=flags.refresh_discovery.is_set(), - indexed=results['indexed'], + indexed=int(results['indexed'] or 0), indexing=flags.refresh_indexing.is_set(), - modeled=results['modeled'], + modeled=int(results['modeled'] or 0), modeling=flags.refresh_modeling.is_set(), cleanup=flags.refresh_cleanup.is_set(), refreshing=flags.refreshing.is_set(), - total_file_groups=results['total_file_groups'], - unindexed=results['unindexed'], + total_file_groups=int(results['total_file_groups'] or 0), + unindexed=int(results['unindexed'] or 0), ) - return status + return progress async def estimate_search(search_str: str, tag_names: List[str]): diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 66b700e1..e775cdda 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -65,17 +65,20 @@ class FileGroup(ModelHelper, Base): __tablename__ = 'file_group' id: int = Column(BigInteger, primary_key=True) + # the Path of the file that can be modeled or indexed. + author = Column(String) # name of the author, maybe even a URL data = Column(FancyJSON) # populated by the modeler files = Column(FancyJSON, nullable=False) # populated during discovery - idempotency = Column(TZDateTime) - indexed = Column(Boolean, default=lambda: False, nullable=False) + idempotency = Column(TZDateTime) # used to track which files need to be deleted during refresh + indexed = Column(Boolean, default=lambda: False, nullable=False) # wrolpi.files.lib.apply_indexers mimetype = Column(String) # wrolpi.files.lib.get_mimetype model = Column(String) # "video", "archive", "ebook", etc. - modification_datetime = Column(TZDateTime) - # the Path of the file that can be modeled or indexed. + modification_datetime = Column(TZDateTime) # the modification date of the file on disk primary_path: pathlib.Path = Column(MediaPathType, nullable=False, unique=True) + published_datetime = Column(TZDateTime) # the date the creator published this file + published_modified_datetime = Column(TZDateTime) # the date the publisher modified this file size = Column(BigInteger, default=lambda: 0) - title = Column(String) + title = Column(String) # user-displayable title tag_files: InstrumentedList = relationship('TagFile', cascade='all') @@ -114,6 +117,9 @@ class FileGroup(ModelHelper, Base): 'tags': tags, 'title': self.title, 'key': self.primary_path, + 'author': self.author, + 'published_datetime': self.published_datetime, + 'published_modified_datetime': self.published_modified_datetime, } return d diff --git a/wrolpi/files/pdfs.py b/wrolpi/files/pdfs.py index ccd723b3..930033b4 100644 --- a/wrolpi/files/pdfs.py +++ b/wrolpi/files/pdfs.py @@ -3,6 +3,9 @@ import dataclasses import pathlib from typing import Generator +import pytz + +from wrolpi import dates from wrolpi.common import register_modeler, logger, truncate_generator_bytes, truncate_object_bytes, \ split_lines_by_length from wrolpi.db import get_db_session @@ -24,31 +27,8 @@ __all__ = ['get_pdf_metadata', 'get_words', 'pdf_modeler'] class PDFMetadata: title: str = None author: str = None - text: str = None - - -def create_index(path): - file_title = Indexer.get_title(path) - - reader = PdfReader(path) - - data = get_pdf_metadata(reader, path) - - if path.stat().st_size > FILE_MAX_PDF_SIZE: - logger.warning(f'PDF too large to fully index: {path}') - return data.title, data.author, file_title, None - - words = '' - try: - # PDFs are complex, don't fail to create title index if text extraction fails. - words = '\n'.join(truncate_generator_bytes(get_words(reader, path), FILE_MAX_TEXT_SIZE)) - words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE) - words = split_lines_by_length(words) - except Exception as e: - logger.error(f'Failed to index {path}', exc_info=e) - if PYTEST: - raise - return data.title, data.author, file_title, words + published_datetime: str = None + modification_datetime: str = None def get_pdf_metadata(reader: PdfReader, path: pathlib.Path) -> PDFMetadata: @@ -60,8 +40,31 @@ def get_pdf_metadata(reader: PdfReader, path: pathlib.Path) -> PDFMetadata: data.title = Indexer.get_title(path) return data - data.title = reader.metadata.title - data.author = reader.metadata.author + if (title := reader.metadata.title) and title.lower() != 'unknown': + data.title = title + if (author := reader.metadata.author) and author.lower() != 'unknown': + data.author = author + + if reader.metadata.creation_date_raw: + try: + str(reader.metadata.creation_date) + data.published_datetime = reader.metadata.creation_date + except ValueError: + # pypdf could not parse the date, lets give it a try... + data.published_datetime = dates.strpdate(reader.metadata.creation_date_raw) + if not data.published_datetime.tzinfo: + data.published_datetime = data.published_datetime.astimezone(pytz.UTC) + + if reader.metadata.modification_date_raw: + try: + str(reader.metadata.modification_date) + data.modification_datetime = reader.metadata.modification_date + except ValueError: + # pypdf could not parse the date, lets give it a try... + data.modification_datetime = dates.strpdate(reader.metadata.modification_date_raw) + if not data.modification_datetime.tzinfo: + data.modification_datetime = data.modification_datetime.astimezone(pytz.UTC) + return data @@ -117,12 +120,32 @@ async def pdf_modeler(): # Assume the first PDF is the only PDF. pdf_file = pdf_files[0]['path'] try: - title, author, file_title, contents = create_index(pdf_file) - file_group.data = {'author': author} - file_group.title = file_group.a_text = title - file_group.b_text = author - file_group.c_text = file_title - file_group.d_text = contents + file_title = Indexer.get_title(pdf_file) + + reader = PdfReader(pdf_file) + + metadata = get_pdf_metadata(reader, pdf_file) + + words = '' + if pdf_file.stat().st_size > FILE_MAX_PDF_SIZE: + logger.warning(f'PDF too large to fully index: {pdf_file}') + else: + try: + # PDFs are complex, don't fail to create title index if text extraction fails. + words = '\n'.join(truncate_generator_bytes(get_words(reader, pdf_file), FILE_MAX_TEXT_SIZE)) + words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE) + words = split_lines_by_length(words) + except Exception as e: + logger.error(f'Failed to index {pdf_file}', exc_info=e) + if PYTEST: + raise + + file_group.title = file_group.a_text = metadata.title or file_title + file_group.author = file_group.b_text = metadata.author + file_group.c_text = file_title # The name of the file may not match the title in the PDF metadata. + file_group.d_text = words or None + file_group.published_datetime = metadata.published_datetime + file_group.published_modified_datetime = metadata.modification_datetime file_group.model = 'pdf' except Exception as e: logger.error(f'Failed to index PDF {pdf_file}', exc_info=e) diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 61394f5a..feb0ee96 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -13,7 +13,7 @@ import pytest from PIL import Image from wrolpi.common import timer -from wrolpi.dates import now, from_timestamp +from wrolpi.dates import now from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile from wrolpi.files import lib, indexers from wrolpi.files.models import FileGroup diff --git a/wrolpi/files/test/test_pdfs.py b/wrolpi/files/test/test_pdfs.py index ca4608a5..c7171b27 100644 --- a/wrolpi/files/test/test_pdfs.py +++ b/wrolpi/files/test/test_pdfs.py @@ -13,8 +13,8 @@ async def test_pdf_title(test_session, example_pdf): """PDF modeler extracts a PDF's title and contents.""" await files_lib.refresh_files() - with mock.patch('wrolpi.files.pdfs.create_index') as mock_create_index: - mock_create_index.side_effect = Exception('Should not index twice') + with mock.patch('wrolpi.files.pdfs.get_pdf_metadata') as mock_get_pdf_metadata: + mock_get_pdf_metadata.side_effect = Exception('Should not index twice') await files_lib.refresh_files() pdf: FileGroup = test_session.query(FileGroup).one() @@ -23,7 +23,11 @@ async def test_pdf_title(test_session, example_pdf): assert pdf.b_text == 'roland' assert pdf.c_text == 'pdf example pdf' assert pdf.d_text and pdf.d_text.startswith('Page one\n') and len(pdf.d_text) == 467 - assert pdf.data and pdf.data.get('author') == 'roland' + # Metadata within the PDF. + assert pdf.author + assert pdf.published_datetime + # The date the file was modified. + assert pdf.modification_datetime @pytest.mark.asyncio diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 7018b70b..615edc5c 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -17,13 +17,13 @@ from sanic_ext.extensions.openapi import openapi from vininfo import Vin from vininfo.details._base import VinDetails -from wrolpi import admin, status, flags, schema +from wrolpi import admin, status, flags, schema, dates from wrolpi import tags from wrolpi.admin import HotspotStatus from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \ wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, LOG_LEVEL, \ set_global_log_level -from wrolpi.dates import now, strptime +from wrolpi.dates import now from wrolpi.downloader import download_manager from wrolpi.errors import WROLModeEnabled, APIError, HotspotError, InvalidDownload, \ HotspotPasswordTooShort, NativeOnly, InvalidConfig @@ -411,7 +411,7 @@ async def get_statistics(_): @validate(query=schema.EventsRequest) async def feed(_: Request, query: schema.EventsRequest): start = now() - after = None if query.after == 'None' else strptime(query.after) + after = None if query.after == 'None' else dates.strpdate(query.after) events = get_events(after) return json_response(dict(events=events, now=start)) diff --git a/wrolpi/scrape_downloader.py b/wrolpi/scrape_downloader.py index bd75f4c4..36c77b4e 100644 --- a/wrolpi/scrape_downloader.py +++ b/wrolpi/scrape_downloader.py @@ -2,9 +2,7 @@ import pathlib from http import HTTPStatus from urllib.parse import urlparse -import bs4 - -from wrolpi.common import aiohttp_session, logger +from wrolpi.common import aiohttp_session, logger, get_html_soup from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError from wrolpi.vars import DEFAULT_HTTP_HEADERS @@ -90,7 +88,7 @@ class ScrapeHTMLDownloader(Downloader): # Find all anchors, search for matching files, and more URLs to search. try: - soup = bs4.BeautifulSoup(content, 'html.parser') + soup = get_html_soup(content) except Exception as e: logger.error(f'Failed to parse HTML from {url}', exc_info=e) continue diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 3a2446c1..09f9f7bb 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -302,28 +302,6 @@ def test_escape_file_name(name, expected): raise AssertionError(f'Escape of {repr(name)}: {repr(result)} != {repr(expected)}') -@pytest.mark.parametrize( - 'paths,suffix_groups,expected', [ - ([], [], ()), - (['foo.mp4', 'foo.info.json'], [('.mp4',)], ('foo.mp4',)), - (['foo.mp4', 'foo.info.json'], [('.mp4',), ('.info.json',)], ('foo.mp4', 'foo.info.json')), - (['foo.mp4', 'foo.info.json'], [('.mp4',), ('.nope',), ('.info.json',)], ('foo.mp4', None, 'foo.info.json')), - (['foo.mp4', 'foo.info.json', 'extra.txt'], [('.mp4',)], ('foo.mp4',)), - (['foo.mp4'], [('.mp4', '.flv'), ('.info.json',)], ('foo.mp4', None)), - ( - # Two files are matched to the closest suffix. - ['foo.info.json', 'bar.json'], - [('.info.json',), ('.json',)], # TODO longest suffix must be first - ('foo.info.json', 'bar.json'), - ), - ] -) -def test_match_paths_to_suffixes(paths, suffix_groups, expected): - paths = [pathlib.Path(i) for i in paths] - expected = tuple(pathlib.Path(i) if i else None for i in expected) - assert (i := common.match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}' - - def test_truncate_object_bytes(): """ Objects can be truncated (lists will be shortened) so they will fit in tsvector columns. diff --git a/wrolpi/test/test_dates.py b/wrolpi/test/test_dates.py index d919beb0..b5539ecc 100644 --- a/wrolpi/test/test_dates.py +++ b/wrolpi/test/test_dates.py @@ -1,7 +1,8 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import pytest import pytz +from dateutil.tz import tzoffset from sqlalchemy import Column, Integer from wrolpi import dates @@ -86,18 +87,54 @@ def test_timedelta_to_timestamp(td, expected): @pytest.mark.parametrize('dt,expected', [ - ('2023-02-26T02:27:26.408944+00:00', datetime(2023, 2, 26, 2, 27, 26, 408944, tzinfo=pytz.UTC)), - ('2023-02-26T02:27:26.408944 00:00', datetime(2023, 2, 26, 2, 27, 26, 408944, tzinfo=pytz.UTC)), -]) -def test_strptime(dt, expected): - assert dates.strptime(dt) == expected - - -@pytest.mark.parametrize('dt,expected', [ - ('20010101', datetime(2001, 1, 1, tzinfo=pytz.UTC)), - ('20050607', datetime(2005, 6, 7, tzinfo=pytz.UTC)), - ('2006-07-08', datetime(2006, 7, 8, tzinfo=pytz.UTC)), - ('2007-8-9', datetime(2007, 8, 9, tzinfo=pytz.UTC)), + ('20010101', datetime(2001, 1, 1)), + ('20050607', datetime(2005, 6, 7)), + ('2006-07-08', datetime(2006, 7, 8)), + ('2007-8-9', datetime(2007, 8, 9)), + ('2023-10-19T05:53:24+00:00', datetime(2023, 10, 19, 5, 53, 24, tzinfo=pytz.UTC)), + ('2022-09-27T00:40:19.000Z', datetime(2022, 9, 27, 0, 40, 19, tzinfo=pytz.UTC)), + ('06/17/2022', datetime(2022, 6, 17)), + ('14.3.2002', datetime(2002, 3, 14)), + ('2002.3.14', datetime(2002, 3, 14)), + ('2022/8/1', datetime(2022, 8, 1)), + ('2022-8-1', datetime(2022, 8, 1)), + ('2023-08-22T02:45:59+0000', datetime(2023, 8, 22, 2, 45, 59, tzinfo=pytz.UTC)), + ('2023-08-29T15:29-0400', datetime(2023, 8, 29, 15, 29, tzinfo=tzoffset(None, -14400))), + ('2023-08-29T15:29+0400', datetime(2023, 8, 29, 15, 29, tzinfo=tzoffset(None, 14400))), + ('2011-11-04', datetime(2011, 11, 4, 0, 0)), + ('20111104', datetime(2011, 11, 4, 0, 0)), + ('2011-11-04T00:05:23', datetime(2011, 11, 4, 0, 5, 23)), + ('2011-11-04T00:05:23Z', datetime(2011, 11, 4, 0, 5, 23, tzinfo=pytz.UTC)), + ('20111104T000523', datetime(2011, 11, 4, 0, 5, 23)), + ('2011-W01-2T00:05:23.283', datetime(2011, 1, 4, 0, 5, 23, 283000)), + ('2011-11-04 00:05:23.283', datetime(2011, 11, 4, 0, 5, 23, 283000)), + ('2011-11-04 00:05:23.283+00:00', datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=pytz.UTC)), + ('2011-11-04T00:05:23+04:00', datetime(2011, 11, 3, 20, 5, 23, tzinfo=pytz.UTC)), + ('2000', datetime(2000, 1, 1, tzinfo=pytz.UTC)), + ('Tuesday, October 19, 1999 3:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)), + ('Tue, October 19, 1999 3:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)), + ('Tue, October 19, 1999 03:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)), + # From SingleFile. + ('Fri Jun 17 2022 19:24:52', datetime(2022, 6, 17, 19, 24, 52)), + # PDFs are the Wild West... + ("D:20221226113758-07'00", datetime(2022, 12, 26, 11, 37, 58, tzinfo=timezone(timedelta(days=-1, seconds=61200)))), + ('D:20200205184724', datetime(2020, 2, 5, 18, 47, 24)), + ("D:20091019120104+", datetime(2009, 10, 19, 12, 1, 4)), ]) def test_strpdate(dt, expected): assert dates.strpdate(dt) == expected + + +def test_invalid_strpdate(): + with pytest.raises(RuntimeError): + dates.strpdate('1') + with pytest.raises(RuntimeError): + dates.strpdate('asdf') + with pytest.raises(RuntimeError): + dates.strpdate('13/1/2008') + + # Not a real date. + with pytest.raises(RuntimeError): + dates.strpdate('2001-2-31') + with pytest.raises(RuntimeError): + dates.strpdate('2001-2-30') diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 1ac1dc32..98a9cadd 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -46,8 +46,6 @@ def truthy_arg(value: str) -> bool: # Get the media directory from the environment. DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi') MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY)) -if not MEDIA_DIRECTORY.is_dir() and not PYTEST: - print(f'Media directory does not exist! {MEDIA_DIRECTORY}', file=sys.stderr) CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config' MODULES_DIR: Path = PROJECT_DIR / 'modules'