;
}
- const {data} = file;
- let author;
- if (data) {
- author = data['author'];
- }
+ const author = file.author;
const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
const color = mimetypeColor(file.mimetype);
const size = file.size !== null && file.size !== undefined ? humanFileSize(file.size) : null;
diff --git a/debian-live-config/config/package-lists/wrolpi.list.chroot b/debian-live-config/config/package-lists/wrolpi.list.chroot
index 5629e395..d016cad5 100644
--- a/debian-live-config/config/package-lists/wrolpi.list.chroot
+++ b/debian-live-config/config/package-lists/wrolpi.list.chroot
@@ -1,5 +1,6 @@
apache2
apache2-dev
+apache2-doc
apt-transport-https
autoconf
build-essential
@@ -17,8 +18,10 @@ fonts-noto-cjk
fonts-noto-hinted
fonts-noto-unhinted
gcc
+gcc-doc
gdal-bin
git
+git-doc
gnupg-agent
hostapd
htop
@@ -64,6 +67,7 @@ osm2pgsql
osmium-tool
pkg-config
postgis
+postgis-doc
postgresql-15
postgresql-15-postgis-3
postgresql-15-postgis-3-scripts
diff --git a/etc/raspberrypios/50x.html b/etc/raspberrypios/50x.html
index 529a1511..5f9dd621 100644
--- a/etc/raspberrypios/50x.html
+++ b/etc/raspberrypios/50x.html
@@ -1,3 +1,4 @@
+
@@ -5,8 +6,8 @@
nginx could not find WROLPi! Are your services running?
-sudo systemctl status wrolpi-app
-sudo systemctl status wrolpi-api
+sudo systemctl status wrolpi-app
+sudo systemctl status wrolpi-api
If either of those services are not running, start all WROLPi services:
@@ -15,71 +16,72 @@
Or, start them individually:
-sudo systemctl start wrolpi-app
-sudo systemctl start wrolpi-api
-sudo systemctl start wrolpi-kiwix
+sudo systemctl start wrolpi-app
+sudo systemctl start wrolpi-api
+sudo systemctl start wrolpi-kiwix
-View the logs
-journalctl -u wrolpi-app
-journalctl -u wrolpi-api
-journalctl -u wrolpi-kiwix
-Watch the logs
-journalctl -fu wrolpi-app
-journalctl -fu wrolpi-api
-journalctl -fu wrolpi-kiwix
+View the logs:
+journalctl -u wrolpi-app
+journalctl -u wrolpi-api
+journalctl -u wrolpi-kiwix
+Watch the logs:
+journalctl -fu wrolpi-app
+journalctl -fu wrolpi-api
+journalctl -fu wrolpi-kiwix
Help script
Run the provided help script which may narrow down the scope of the problem.
-bash /opt/wrolpi/help.sh
+bash /opt/wrolpi/help.sh
Repair script
Run the provided repair script which will attempt to automatically repair the WROLPi systems.
-bash /opt/wrolpi/repair.sh
+bash /opt/wrolpi/repair.sh
Still not working?
You can view the media directory by navigating to /media
+The API may be available by navigating to port 8081
+The map may be available by navigating to port 8084
+Kiwix may be available by navigating to port 8085
-The API may be available by navigating to port 8081
+
-The map may be available by navigating to port 8084
-
-Kiwix may be available by navigating to port 8085
+Third-Party Documentation
+WROLPi comes with official documentation from some third-party applications:
+
diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py
index 3a80c8b5..3f33e3a5 100644
--- a/modules/archive/__init__.py
+++ b/modules/archive/__init__.py
@@ -238,20 +238,21 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
return archive
except Exception as e:
- logger.error(f'Failed to model Archive {file_group}', exc_info=e)
- if PYTEST:
- raise
+ logger.error('Failed to model Archive', exc_info=e)
+ raise InvalidArchive(f'Failed to model Archive {file_group}') from e
@register_modeler
async def archive_modeler():
+ """Searches DB for FileGroups that contain an HTML file. If the HTML file is a SingleFile, we model it as an
+ Archive."""
invalid_archives = set()
while True:
with get_db_session(commit=True) as session:
results = session.query(FileGroup, Archive) \
.filter(
- # Get all groups that contain a PDF that have not been indexed.
+ # Get all groups that contain an HTML file that have not been indexed.
FileGroup.indexed != True,
FileGroup.mimetype == 'text/html',
).filter(not_(FileGroup.id.in_(list(invalid_archives)))) \
@@ -262,7 +263,8 @@ async def archive_modeler():
for file_group, archive in results:
processed += 1
- with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}', logger__=logger):
+ with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}',
+ logger__=logger):
if archive:
archive: Archive
try:
@@ -276,7 +278,7 @@ async def archive_modeler():
try:
model_archive(file_group, session=session)
except InvalidArchive:
- # May not be a real Singlefile archive.
+ # It was not a real Archive. Many HTML files will not be an Archive.
pass
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py
index 5c82e08e..d8965646 100644
--- a/modules/archive/conftest.py
+++ b/modules/archive/conftest.py
@@ -76,7 +76,8 @@ def archive_factory(test_session, archive_directory, make_files_structure):
archive = Archive.from_paths(test_session, *files)
archive.url = url
archive.title = title
- archive.archive_datetime = next(now)
+ archive.archive_datetime = archive.file_group.published_datetime = next(now)
+ archive.file_group.modification_datetime = next(now)
archive.domain = domain
archive.validate()
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 04f26d11..74c4b246 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -5,19 +5,22 @@ import pathlib
import re
from dataclasses import dataclass
from datetime import datetime
-from typing import Optional, Tuple, List
+from typing import Optional, Tuple, List, Union
+from urllib.parse import urlparse
-from bs4 import BeautifulSoup
+import pytz
from sqlalchemy import asc
from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
+from wrolpi import dates
from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \
- format_html_string, split_lines_by_length
+ format_html_string, split_lines_by_length, get_html_soup
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy
from wrolpi.tags import tag_names_to_file_group_sub_select
+from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@@ -220,13 +223,157 @@ def get_title_from_html(html: str, url: str = None) -> str:
"""
Try and get the title from the
"""
- soup = BeautifulSoup(html, features='html.parser')
+ soup = get_html_soup(html)
try:
return soup.title.string
except Exception: # noqa
logger.debug(f'Unable to extract title {url}')
+SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n')
+
+
+def get_url_from_singlefile(html: bytes) -> str:
+ """Extract URL from SingleFile contents."""
+ html = html.decode()
+ if SINGLEFILE_HEADER not in html:
+ raise RuntimeError(f'Not a singlefile!')
+
+ url, = SINGLEFILE_URL_EXTRACTOR.findall(html)
+ url = url.strip()
+
+ if not url.startswith('http'):
+ raise RuntimeError('URL did not start with http')
+
+ # Verify that the URL can be parsed.
+ urlparse(url)
+ return url
+
+
+@dataclass
+class ArticleMetadata:
+ title: str = None
+ published_datetime: datetime = None
+ modified_datetime: datetime = None
+ description: str = None
+ author: str = None
+
+
+def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True) -> ArticleMetadata:
+ """
+ Read the data from the tags, extract any data relevant to the article. This function also reads the
+
+'''
singlefile, *_ = make_files_structure({
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
@@ -543,8 +553,14 @@ async def test_refresh_archives_index(test_session, make_files_structure):
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
- assert archive.file_group.a_text == 'the title'
- assert archive.file_group.d_text == 'article text contents'
+
+ assert archive.file_group.author == 'AUTHOR NAME'
+ assert archive.file_group.published_datetime
+ assert archive.file_group.published_modified_datetime
+
+ assert archive.file_group.a_text == 'the title' # from readability json
+ assert archive.file_group.b_text == 'The article description' # From application/ld+json
+ assert archive.file_group.d_text == 'article text contents' # from readability txt
assert archive.file_group.indexed is True
@@ -651,3 +667,143 @@ def test_archive_history(test_session, archive_factory):
assert not archive3.history, 'archive3 has no history'
assert not archive4.history, 'archive4 has no history'
assert not archive5.history, 'archive5 has no history'
+
+
+SINGLEFILE_1 = b'''
+
+
+
+'''
+
+SINGLEFILE_2 = b'''
+
+
+
+
+ This is not a meta title and should be ignored
+
+
+
+
+
+
+
+
+'''
+
+SINGLEFILE_3 = b'''
+
+
+This is not a meta title and should be ignored
+
+'''
+
+SINGLEFILE_4 = b'''
+
+
+This is not a meta title and should be ignored
+
+ Link Author
+
+'''
+
+SINGLEFILE_5 = b'''
+
+
+This is not a meta title and should be ignored
+
+ The Span Author
+
+
+
+'''
+
+
+def test_parse_article_html_metadata():
+ """Singlefiles may contain metadata tags."""
+ # Metadata can be empty.
+ metadata = lib.parse_article_html_metadata(SINGLEFILE_1)
+ assert metadata.title is None
+ assert metadata.published_datetime is None
+ assert metadata.modified_datetime is None
+ assert metadata.description is None
+ assert metadata.author is None
+
+ # Data from elements.
+ metadata = lib.parse_article_html_metadata(SINGLEFILE_2)
+ assert metadata.title == 'The Title'
+ assert metadata.published_datetime == datetime(2023, 10, 18, 4, 52, 23, tzinfo=utc)
+ assert metadata.modified_datetime == datetime(2023, 10, 19, 5, 53, 24, tzinfo=utc)
+ assert metadata.description is None
+ assert metadata.author == 'Author Name'
+
+ # Data from Article structured data
+ # https://developers.google.com/search/docs/appearance/structured-data/article
+ metadata = lib.parse_article_html_metadata(SINGLEFILE_3)
+ assert metadata.title == 'The headline'
+ assert metadata.published_datetime == datetime(2022, 9, 27, 0, 40, 19, tzinfo=utc)
+ assert metadata.modified_datetime == datetime(2022, 9, 27, 13, 43, 47, 971000, tzinfo=utc)
+ assert metadata.description == 'The article description'
+ assert metadata.author == 'BOBBY'
+
+ # Data from author anchor element.
+ metadata = lib.parse_article_html_metadata(SINGLEFILE_4)
+ assert metadata.author == 'Link Author'
+
+ # author anchor may be a list.
+ metadata = lib.parse_article_html_metadata(SINGLEFILE_5)
+ assert metadata.author == 'The Span Author'
+ assert metadata.published_datetime == datetime(2023, 8, 25, tzinfo=pytz.UTC)
+
+
+@pytest.mark.parametrize(
+ 'html,expected',
+ [
+ ('',
+ datetime(2023, 10, 18, 4, 52, 23, tzinfo=utc)),
+ ('March 17, 2022',
+ datetime(2022, 3, 17, 10, 0, tzinfo=pytz.UTC)),
+ ('',
+ datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)),
+ ('',
+ datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)),
+ (
+ '3/05/2013 12:12:00 AM',
+ datetime(2013, 3, 5, 6, 12, tzinfo=pytz.UTC)),
+ ('',
+ datetime(2023, 8, 25, tzinfo=pytz.UTC))
+ ])
+def test_published_datetime(html, expected):
+ assert lib.parse_article_html_metadata(html).published_datetime == expected
+
+
+@pytest.mark.parametrize(
+ 'html,expected',
+ [
+ ('',
+ datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)),
+ ('',
+ datetime(2023, 10, 19, 5, 53, 24, tzinfo=utc)),
+ (
+ '',
+ datetime(2022, 9, 27, 13, 43, 47, 971000, tzinfo=utc)),
+ ])
+def test_modified_datetime(html, expected):
+ assert lib.parse_article_html_metadata(html).modified_datetime == expected
+
+
+@pytest.mark.parametrize(
+ 'html,expected',
+ [
+ ('Linda', 'Linda'),
+ ('', 'Billy'),
+ ]
+)
+def test_author(html, expected):
+ assert lib.parse_article_html_metadata(html).author == expected
diff --git a/modules/inventory/common.py b/modules/inventory/common.py
index 2990a6e5..a8020b8a 100644
--- a/modules/inventory/common.py
+++ b/modules/inventory/common.py
@@ -9,8 +9,8 @@ from wrolpi import before_startup
from wrolpi.common import logger, Base, ConfigFile
from wrolpi.db import get_db_session
from .errors import NoInventories, InventoriesVersionMismatch
-from .inventory import unit_registry, get_items, get_inventories, increment_inventories_version, \
- get_inventories_version
+from .inventory import get_items, get_inventories, increment_inventories_version, \
+ get_inventories_version, get_unit_registry
from .models import Inventory, Item
MY_DIR: Path = Path(__file__).parent
@@ -23,6 +23,8 @@ def sum_by_key(items: List[Base], key: callable):
Sum the total size of each item by the provided key function. Returns a dict containing the key, and the total_size
for that key. Combine total of like units. This means ounces and pounds will be in the same total.
"""
+ unit_registry = get_unit_registry()
+
summed = dict()
for item in items:
k = key(item)
@@ -61,17 +63,28 @@ def get_inventory_by_keys(keys: Tuple, inventory_id: int):
get_inventory_by_category = partial(get_inventory_by_keys, ('category',))
get_inventory_by_subcategory = partial(get_inventory_by_keys, ('category', 'subcategory'))
get_inventory_by_name = partial(get_inventory_by_keys, ('brand', 'name'))
-INVENTORY_UNITS = {
- ('ounce', 1): (16, unit_registry.pound),
- ('pound', 1): (2000, unit_registry.ton),
-}
+INVENTORY_UNITS: dict = None
UNIT_PRECISION = 5
-def compact_unit(quantity: unit_registry.Quantity) -> unit_registry.Quantity:
+def initialize_inventory_units() -> dict:
+ global INVENTORY_UNITS
+ if INVENTORY_UNITS:
+ return INVENTORY_UNITS
+
+ unit_registry = get_unit_registry()
+ INVENTORY_UNITS = {
+ ('ounce', 1): (16, unit_registry.pound),
+ ('pound', 1): (2000, unit_registry.ton),
+ }
+
+
+def compact_unit(quantity: Quantity) -> Quantity:
"""
Convert a Quantity to it's more readable format. Such as 2000 pounds to 1 ton.
"""
+ initialize_inventory_units()
+
number, (units,) = quantity.to_tuple()
next_unit = INVENTORY_UNITS.get(units)
if not next_unit:
@@ -88,7 +101,8 @@ def compact_unit(quantity: unit_registry.Quantity) -> unit_registry.Quantity:
return quantity
-def quantity_to_tuple(quantity: unit_registry.Quantity) -> Tuple[Decimal, Quantity]:
+def quantity_to_tuple(quantity: Quantity) -> Tuple[Decimal, Quantity]:
+ unit_registry = get_unit_registry()
decimal, (units,) = quantity.to_tuple()
unit, _ = units
quantity, unit = round(decimal, UNIT_PRECISION), unit_registry(unit)
diff --git a/modules/inventory/inventory.py b/modules/inventory/inventory.py
index 6705dd6b..91208440 100644
--- a/modules/inventory/inventory.py
+++ b/modules/inventory/inventory.py
@@ -15,7 +15,38 @@ from .models import Inventory, Item, InventoriesVersion
logger = logger.getChild(__name__)
-unit_registry = pint.UnitRegistry()
+__all__ = [
+ 'DEFAULT_CATEGORIES',
+ 'DEFAULT_INVENTORIES',
+ 'delete_inventory',
+ 'delete_items',
+ 'get_brands',
+ 'get_categories',
+ 'get_inventories',
+ 'get_inventories_version',
+ 'get_items',
+ 'get_next_inventories_version',
+ 'get_unit_registry',
+ 'increment_inventories_version',
+ 'logger',
+ 'save_inventory',
+ 'save_item',
+ 'sort_categories',
+ 'update_inventory',
+ 'update_item',
+]
+
+UNIT_REGISTRY: pint.UnitRegistry = None
+
+
+def get_unit_registry() -> pint.UnitRegistry:
+ global UNIT_REGISTRY
+ if UNIT_REGISTRY:
+ return UNIT_REGISTRY
+
+ UNIT_REGISTRY = pint.UnitRegistry()
+ return UNIT_REGISTRY
+
DEFAULT_CATEGORIES = [
('salt', 'cooking ingredients'),
diff --git a/modules/inventory/test/test_inventory.py b/modules/inventory/test/test_inventory.py
index 8a234073..9d2e9565 100644
--- a/modules/inventory/test/test_inventory.py
+++ b/modules/inventory/test/test_inventory.py
@@ -15,11 +15,12 @@ from .. import init
from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \
compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file, get_inventories_config
from ..errors import NoInventories, InventoriesVersionMismatch
-from ..inventory import unit_registry, \
- get_inventories, save_inventory, update_inventory, \
- delete_inventory, get_categories
+from ..inventory import get_inventories, save_inventory, update_inventory, \
+ delete_inventory, get_categories, get_unit_registry
from ..models import Item, Inventory
+UNIT_REGISTRY = get_unit_registry()
+
TEST_ITEMS_COLUMNS = (
'inventory_id',
'brand',
@@ -199,11 +200,11 @@ def quantity_to_string(quantity: Quantity) -> str:
@pytest.mark.parametrize(
'quantity,expected',
[
- (Decimal(5) * unit_registry.ounce, '5 ounce'),
- (Decimal(16) * unit_registry.ounce, '1 pound'),
- (Decimal(500) * unit_registry.pound, '500 pound'),
- (Decimal(2000) * unit_registry.pound, '1 ton'),
- (Decimal(128000) * unit_registry.ounce, '4 ton'),
+ (Decimal(5) * UNIT_REGISTRY.ounce, '5 ounce'),
+ (Decimal(16) * UNIT_REGISTRY.ounce, '1 pound'),
+ (Decimal(500) * UNIT_REGISTRY.pound, '500 pound'),
+ (Decimal(2000) * UNIT_REGISTRY.pound, '1 ton'),
+ (Decimal(128000) * UNIT_REGISTRY.ounce, '4 ton'),
]
)
def test_compact_unit(quantity, expected):
@@ -211,7 +212,7 @@ def test_compact_unit(quantity, expected):
assert quantity_to_string(compact_unit(quantity)) == expected
-pound, oz, gram, gallon = unit_registry('pound'), unit_registry('oz'), unit_registry('gram'), unit_registry('gallon')
+pound, oz, gram, gallon = UNIT_REGISTRY('pound'), UNIT_REGISTRY('oz'), UNIT_REGISTRY('gram'), UNIT_REGISTRY('gallon')
pound, oz, gram, gallon = pound.units, oz.units, gram.units, gallon.units
mass = pound.dimensionality
length = gallon.dimensionality
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 3530b7d7..56671428 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -6,6 +6,7 @@ import re
import warnings
from typing import Tuple, Optional, Generator
+import pytz
from sqlalchemy import or_
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
@@ -102,6 +103,8 @@ def validate_video(video: Video, channel_generate_poster: bool):
upload_date = upload_date or video.upload_date
if upload_date and isinstance(upload_date, str):
upload_date = dates.strpdate(upload_date)
+ if not upload_date.tzinfo:
+ upload_date = upload_date.astimezone(pytz.UTC)
video.file_group.title = video.file_group.title or html.unescape(title)
video.upload_date = upload_date
video.source_id = video.source_id or source_id
@@ -120,7 +123,7 @@ def validate_video(video: Video, channel_generate_poster: bool):
if video.ffprobe_json:
if duration := video.ffprobe_json['format'].get('duration'):
video.duration = float(duration)
- elif video_streams := video.get_streams_by_codec_type('video'):
+ elif (video_streams := video.get_streams_by_codec_type('video')) and 'duration' in video_streams[0]:
video.duration = float(video_streams[0]['duration'])
else:
# Slowest method.
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index a4360cb0..1e6f3c21 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -4,7 +4,6 @@ from copy import copy
from unittest import mock
import pytest
-from yt_dlp.utils import UnsupportedError
from modules.videos.channel.lib import download_channel
from modules.videos.downloader import VideoDownloader, \
diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py
index 5248f08a..a786358b 100644
--- a/modules/videos/test/test_lib.py
+++ b/modules/videos/test/test_lib.py
@@ -1,4 +1,3 @@
-import datetime
import pathlib
import shutil
@@ -7,10 +6,8 @@ import pytest
from modules.videos import lib
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
from modules.videos.models import Video
-from wrolpi.dates import now
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
-from wrolpi.tags import Tag
from wrolpi.vars import PROJECT_DIR
@@ -67,7 +64,7 @@ async def test_video_factory(test_session, video_factory, channel_factory):
assert video.source_id == 'some id'
-def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file):
+def test_validate_video(test_directory, video_factory, image_bytes_factory, video_file):
"""A video poster will be generated only if the channel permits."""
video_file = video_file.rename(test_directory / 'Channel Name_20050607_1234567890_The Title.mp4')
vid1 = video_factory(with_video_file=video_file)
@@ -81,7 +78,8 @@ def test_validate_video(test_session, test_directory, video_factory, image_bytes
assert vid1.poster_path.is_file(), 'Poster path does not exist'
assert vid1.video_path.stem == vid1.poster_path.stem
assert vid1.poster_path.suffix == '.jpg'
- assert vid1.upload_date == datetime.datetime(2005, 6, 7, 0, 0, tzinfo=datetime.timezone.utc)
+ # File name date is assumed to be local timezone.
+ assert vid1.upload_date and vid1.upload_date.year == 2005
assert vid1.source_id == '1234567890'
assert vid1.file_group.title == 'The Title'
diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py
index 4d6d7ad5..1619f466 100644
--- a/modules/videos/video/test/test_api.py
+++ b/modules/videos/video/test/test_api.py
@@ -6,7 +6,6 @@ from json import dumps
import pytest
from modules.videos.models import Video
-from modules.videos.video.lib import get_video_for_app
from wrolpi.dates import now
diff --git a/modules/zim/downloader.py b/modules/zim/downloader.py
index d75b9a45..86f77839 100644
--- a/modules/zim/downloader.py
+++ b/modules/zim/downloader.py
@@ -1,10 +1,8 @@
from http import HTTPStatus
from typing import List
-import bs4
-
from modules.zim import lib
-from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task
+from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task, get_html_soup
from wrolpi.downloader import Downloader, Download, DownloadResult
__all__ = ['KiwixCatalogDownloader', 'KiwixZimDownloader', 'kiwix_zim_downloader', 'kiwix_catalog_downloader']
@@ -20,7 +18,7 @@ async def fetch_hrefs(url: str) -> List[str]:
if status != HTTPStatus.OK:
raise RuntimeError(f'Failed to fetch file catalog')
- soup = bs4.BeautifulSoup(content, 'html.parser')
+ soup = get_html_soup(content)
downloads = list()
for a_ in soup.find_all('a'):
downloads.append(a_['href'])
diff --git a/pi-gen/stage2/04-wrolpi/01-packages b/pi-gen/stage2/04-wrolpi/01-packages
index 3372d805..d35e5d4e 100644
--- a/pi-gen/stage2/04-wrolpi/01-packages
+++ b/pi-gen/stage2/04-wrolpi/01-packages
@@ -1,5 +1,6 @@
apache2
apache2-dev
+apache2-doc
apt-transport-https
autoconf
build-essential
@@ -17,8 +18,10 @@ fonts-noto-cjk
fonts-noto-hinted
fonts-noto-unhinted
gcc
+gcc-doc
gdal-bin
git
+git-doc
gnupg-agent
hostapd
iotop
@@ -62,6 +65,7 @@ osm2pgsql
osmium-tool
pkg-config
postgis
+postgis-doc
postgresql-15
postgresql-15-postgis-3
postgresql-15-postgis-3-scripts
diff --git a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh
index 8a9d7b5c..1a4442dc 100755
--- a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh
+++ b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh
@@ -43,7 +43,7 @@ chmod 644 /var/www/html/*
# Allow wrolpi user to delete map tiles.
usermod -aG _renderd wrolpi
-mkdir /var/cache/renderd/tiles
+[ ! -d /var/cache/renderd/tiles ] && mkdir /var/cache/renderd/tiles
chown -R _renderd:_renderd /var/cache/renderd/tiles
# Create the media directory for the wrolpi user.
diff --git a/requirements.txt b/requirements.txt
index f1fe90af..3280531a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
Pillow==10.0.1
Pint==0.21
-pypdf==3.12.1
PyYAML==6.0
SQLAlchemy[asyncio]==1.3.22
aiohttp==3.8.5
@@ -13,8 +12,10 @@ libzim==3.1.0
mock==5.0.2
psutil==5.9.5
psycopg2-binary==2.9.6
+pypdf==3.12.1
pytest-asyncio==0.21.0
pytest==7.3.1
+python-dateutil==2.8.2
python-dotenv==1.0.0
python-magic~=0.4.27
pytz==2023.3
diff --git a/scripts/singlefile_to_archive.py b/scripts/singlefile_to_archive.py
new file mode 100755
index 00000000..894c03f2
--- /dev/null
+++ b/scripts/singlefile_to_archive.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Extracts Readability, screenshot, etc. from a Singlefile."""
+import argparse
+import json
+import logging
+import pathlib
+import subprocess
+import sys
+import tempfile
+
+from modules.archive import lib as archive_lib
+from wrolpi.cmd import READABILITY_BIN
+from wrolpi.common import html_screenshot, format_html_string, logger
+
+logger = logger.getChild('singlefile_to_archive')
+
+
+def do_readability(html: bytes, url: str) -> dict:
+ """Extract the readability dict from the provided HTML."""
+ with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
+ fh.write(html)
+
+ cmd = (READABILITY_BIN, fh.name, url)
+ output = subprocess.check_output(cmd, timeout=60)
+
+ readability = json.loads(output)
+ return readability
+
+
+def singlefile_to_archive_files(singlefile: pathlib.Path):
+ """Extract Readability files and Screenshot file next to the provided Singlefile."""
+ contents = singlefile.read_bytes()
+
+ readability_html_path = singlefile.with_suffix('.readability.html')
+ readability_json_path = singlefile.with_suffix('.readability.json')
+ readability_text_path = singlefile.with_suffix('.readability.txt')
+
+ # Only extract Readability if any of the files are missing.
+ readability = dict()
+ if not readability_html_path.is_file() or not readability_text_path.is_file() or not readability_json_path.is_file():
+ url = archive_lib.get_url_from_singlefile(contents)
+ try:
+ readability = do_readability(contents, url)
+ except Exception as e:
+ logger.error(f'Failed to run readability', exc_info=e)
+
+ # The real Readability the user wants.
+ readability_html = readability.pop('content', None)
+ if readability and not readability_html_path.is_file() and readability_html:
+ try:
+ readability_html = format_html_string(readability_html)
+ readability_html_path.write_text(readability_html)
+ except Exception as e:
+ logger.error(f'Failed to extract readability: {singlefile}', exc_info=e)
+
+ # The text from within the Readability, used for searching.
+ readability_text = readability.pop('textContent', None)
+ if readability and not readability_text_path.is_file() and readability_text:
+ readability_text_path.write_text(readability_text)
+
+ # Write the JSON last now that content/textContent have been removed.
+ if readability and not readability_json_path.is_file():
+ try:
+ with readability_json_path.open('wt') as fh:
+ fh.write(json.dumps(readability, indent=2))
+ except Exception as e:
+ logger.error(f'Failed to extract Readability text: {singlefile}', exc_info=e)
+
+ # Only generate a screenshot if the file is missing.
+ screenshot_path = singlefile.with_suffix('.png')
+ if not screenshot_path.is_file():
+ try:
+ screenshot = html_screenshot(contents)
+ screenshot_path.write_bytes(screenshot)
+ except Exception as e:
+ logger.error(f'Failed to create screenshot: {singlefile}', exc_info=e)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('files', nargs='+')
+ parser.add_argument('-v', action='count')
+ args = parser.parse_args()
+
+ if args.v == 0:
+ logger.setLevel(logging.WARNING)
+ elif args.v == 1:
+ logger.setLevel(logging.INFO)
+ elif args.v >= 2:
+ logger.setLevel(logging.DEBUG)
+ logger.debug(f'Debug logging')
+
+ if not READABILITY_BIN:
+ print(f'Cannot find "readability-extractor". Install it.')
+ sys.exit(1)
+
+ files = [pathlib.Path(i) for i in args.files]
+ invalid_files = [i for i in files if not archive_lib.is_singlefile_file(i)]
+ if invalid_files:
+ for i in invalid_files:
+ print(f'File is not a singlefile: {i}')
+ sys.exit(1)
+
+ for file in files:
+ singlefile_to_archive_files(file)
+ print(f'Finished {file}')
diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py
index 08b75a3f..382ab957 100644
--- a/wrolpi/cmd.py
+++ b/wrolpi/cmd.py
@@ -55,25 +55,22 @@ CPUFREQ_SET_BIN = which('cpufreq-set', '/usr/bin/cpufreq-set')
WGET_BIN = which('wget', '/usr/bin/wget', flag=flags.wget_installed)
# Map
-BASH_BIN = which('bash', '/bin/bash', warn=True)
+BASH_BIN = which('bash', '/bin/bash')
# Archive
SINGLE_FILE_BIN = which('single-file',
'/usr/bin/single-file', # rpi os
'/usr/local/bin/single-file', # debian
- warn=True,
flag=flags.singlefile_installed,
)
CHROMIUM = which('chromium-browser', 'chromium',
'/usr/bin/chromium-browser', # rpi os
'/usr/bin/chromium', # debian
- warn=True,
flag=flags.chromium_installed,
)
READABILITY_BIN = which('readability-extractor',
'/usr/bin/readability-extractor', # rpi os
'/usr/local/bin/readability-extractor', # debian
- warn=True,
flag=flags.readability_installed,
)
@@ -82,8 +79,7 @@ YT_DLP_BIN = which(
'yt-dlp',
'/usr/local/bin/yt-dlp', # Location in docker container
'/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location
- warn=True,
flag=flags.yt_dlp_installed,
)
-FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe', warn=True, flag=flags.ffprobe_installed)
+FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe', flag=flags.ffprobe_installed)
FFMPEG_BIN = which('ffmpeg', '/usr/bin/ffmpeg', flag=flags.ffmpeg_installed)
diff --git a/wrolpi/common.py b/wrolpi/common.py
index ab231200..1545bde6 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -27,6 +27,7 @@ from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Gener
from urllib.parse import urlparse, urlunsplit
import aiohttp
+import bs4
import yaml
from aiohttp import ClientResponse
from bs4 import BeautifulSoup
@@ -36,7 +37,6 @@ from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
-from wrolpi import dates
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS
@@ -129,8 +129,8 @@ __all__ = [
'get_config',
'get_download_info',
'get_files_and_directories',
+ 'get_html_soup',
'get_media_directory',
- 'get_model_by_table_name',
'get_relative_to_media_directory',
'get_warn_once',
'insert_parameter',
@@ -138,7 +138,6 @@ __all__ = [
'limit_concurrent',
'logger',
'make_media_directory',
- 'match_paths_to_suffixes',
'minimize_dict',
'native_only',
'partition',
@@ -147,7 +146,6 @@ __all__ = [
'register_refresh_cleanup',
'remove_whitespace',
'run_after',
- 'sanitize_link',
'set_log_level',
'set_test_config',
'set_test_media_directory',
@@ -161,6 +159,8 @@ __all__ = [
'wrol_mode_check',
'wrol_mode_enabled',
'zig_zag',
+ 'html_screenshot',
+ 'format_html_string',
]
# Base is used for all SQLAlchemy models.
@@ -200,13 +200,6 @@ class ModelHelper:
Session.object_session(self).flush([self])
-def get_model_by_table_name(table_name):
- """Find a model by its table name."""
- for klass in Base._decl_class_registry.values():
- if hasattr(klass, '__tablename__') and klass.__tablename__ == table_name:
- return klass
-
-
class tsvector(types.TypeDecorator):
impl = types.UnicodeText
@@ -219,12 +212,6 @@ def compile_tsvector(element, compiler, **kw):
URL_CHARS = string.ascii_lowercase + string.digits
-def sanitize_link(link: str) -> str:
- """Remove any non-url safe characters, all will be lowercase."""
- new_link = ''.join(i for i in str(link).lower() if i in URL_CHARS)
- return new_link
-
-
class ConfigFile:
"""
This class keeps track of the contents of a config file. You can update the config (and it's file) by
@@ -1118,37 +1105,21 @@ def register_refresh_cleanup(func: callable):
async def apply_refresh_cleanup():
+ # TODO convert all functions to async so refresh functions can be used.
for func in REFRESH_CLEANUP:
- start = dates.now()
- try:
- logger_.info(f'Applying refresh cleanup {func.__name__}')
- func()
- except Exception as e:
- logger_.error(f'Refresh cleanup {func.__name__} failed!', exc_info=e)
- if PYTEST:
- raise
- finally:
- elapsed = (dates.now() - start).total_seconds()
- logger_.warning(f'After refresh cleanup {func.__name__} took {int(elapsed)} seconds')
+ slow_message = f'After refresh cleanup {func.__name__} took %(elapsed)s seconds'
+ with slow_logger(5, slow_message, logger__=logger_):
+ try:
+ logger_.info(f'Applying refresh cleanup {func.__name__}')
+ func()
+ except Exception as e:
+ logger_.error(f'Refresh cleanup {func.__name__} failed!', exc_info=e)
+ if PYTEST:
+ raise
# Sleep to catch cancel.
await asyncio.sleep(0)
-@iterify(tuple)
-def match_paths_to_suffixes(paths: List[pathlib.Path], suffix_groups: List[Tuple[str]]):
- paths = paths.copy()
- for group in suffix_groups:
- match = None
- for suffix in group:
- # Compare all suffixes in the group to the path's name. Yield the first path that shares the first
- # suffix. If no suffix matches any path, yield None.
- match = next(filter(lambda i: i.name.endswith(suffix), paths), None)
- if match:
- paths.pop(paths.index(match))
- break
- yield match
-
-
def chunks(it: Iterable, size: int):
"""Split an iterable into iterables of the defined length."""
it = iter(it)
@@ -1210,7 +1181,7 @@ def timer(name):
@contextlib.contextmanager
-def slow_logger(max_seconds: int, message: str, logger__=logger, level=logging.WARNING):
+def slow_logger(max_seconds: int, message: str, logger__=logger_, level=logging.WARNING):
"""Only logs when the context duration exceeds the `max_seconds`."""
before = now()
try:
@@ -1472,8 +1443,13 @@ def url_strip_host(url: str) -> str:
return url or '/'
-def extract_html_text(html: str) -> str:
+def get_html_soup(html: Union[bytes, str]) -> bs4.BeautifulSoup:
soup = BeautifulSoup(html, features='html.parser')
+ return soup
+
+
+def extract_html_text(html: str) -> str:
+ soup = get_html_soup(html)
# kill all script and style elements
for script in soup(["script", "style"]):
@@ -1538,6 +1514,9 @@ def format_json_file(file: pathlib.Path, indent: int = 2):
def format_html_string(html: str) -> str:
+ if not html.strip():
+ raise RuntimeError('Refusing to format empty HTMl string.')
+
soup = BeautifulSoup(html, features='lxml')
return soup.prettify()
diff --git a/wrolpi/dates.py b/wrolpi/dates.py
index 7daec46c..593091ad 100644
--- a/wrolpi/dates.py
+++ b/wrolpi/dates.py
@@ -1,8 +1,10 @@
import logging
+import re
from datetime import datetime, timezone, timedelta
from enum import Enum
from typing import Union
+import dateutil.parser
import pytz
from sqlalchemy import types
@@ -41,18 +43,84 @@ def strftime(dt: datetime) -> str:
return dt.isoformat()
-def strptime(dt: str) -> datetime:
- if isinstance(dt, str) and dt.endswith(' 00:00'):
- # URL decoding removed +
- dt = f'{dt[:26]}+00:00'
- return datetime.fromisoformat(dt).astimezone(pytz.UTC)
-
-
def strpdate(dt: str) -> datetime:
- if '-' in dt:
- return datetime.strptime(dt, '%Y-%m-%d').replace(tzinfo=pytz.UTC)
+ """
+ Attempt to parse a datetime string. Tries to find a Timezone, if possible. DB requires timezone.
- return datetime.strptime(dt, '%Y%m%d').replace(tzinfo=pytz.UTC)
+ TODO replace this in Python 3.11 https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat
+ """
+ try:
+ if dt.count('/') == 2:
+ # No timezone info.
+ try:
+ a, b, c = dt.split('/')
+ if len(c) == 4:
+ # Assume m/d/Y
+ return datetime.strptime(dt, '%m/%d/%Y')
+ if len(a) == 4:
+ # Y/m/d
+ return datetime.strptime(dt, '%Y/%m/%d')
+ except ValueError:
+ pass
+ elif dt.count('-') == 2 and len(dt) <= 10:
+ # No timezone info.
+ a, b, c = dt.split('-')
+ if len(a) == 4:
+ # Y-m-d
+ return datetime.strptime(dt, '%Y-%m-%d')
+
+ if dt.startswith('D:'):
+ # D:20221226113758-07'00, D:20200205184724, etc.
+ dt2 = dt.replace("'", '').rstrip('+')
+ try:
+ return datetime.strptime(dt2, "D:%Y%m%d%H%M%S%z")
+ except ValueError:
+ pass
+
+ return datetime.strptime(dt2, "D:%Y%m%d%H%M%S")
+
+ if dt.count('-') == 0 and dt.count('/') == 0 and dt.isdigit() and len(dt) == 8:
+ # Assume %Y%m%d date
+ return datetime.strptime(dt, '%Y%m%d')
+
+ if dt.count('.') == 2 and len(dt) <= 9:
+ # yyyy.mm.dd or dd.mm.yyyy
+ try:
+ a, b, c = dt.split('.')
+ if len(a) == 4:
+ return datetime.strptime(dt, '%Y.%m.%d')
+ if len(c) == 4:
+ return datetime.strptime(dt, '%d.%m.%Y')
+ except ValueError:
+ pass
+
+ try:
+ # Fri Jun 17 2022 19:24:52 (from Singlefile)
+ return datetime.strptime(dt, '%a %b %d %Y %H:%M:%S')
+ except ValueError:
+ pass
+
+ if len(dt) == 4 and dt.isdigit():
+ # Assume this is a year
+ return datetime.strptime(dt, '%Y').replace(tzinfo=pytz.UTC)
+
+ if ',' in dt and (dt.endswith('AM') or dt.endswith('PM')):
+ try:
+ # Tuesday, October 19, 1999 3:41:01 PM
+ return datetime.strptime(dt, '%A, %B %d, %Y %I:%M:%S %p')
+ except ValueError:
+ pass
+
+ try:
+ # Tue, October 19, 1999 3:41:01 PM
+ return datetime.strptime(dt, '%a, %B %d, %Y %I:%M:%S %p')
+ except ValueError:
+ pass
+
+ # Last, try third-party module to parse ISO 8601 datetime.
+ return dateutil.parser.isoparse(dt)
+ except Exception as e:
+ raise RuntimeError(f'Unable to parse datetime string: {dt}') from e
def strftime_ms(dt: datetime) -> str:
@@ -73,6 +141,7 @@ def seconds_to_timestamp(seconds: Union[int, float]) -> str:
>>> seconds_to_timestamp(5)
# '00:00:05'
>>> seconds_to_timestamp(60)
+
# '00:01:00'
>>> seconds_to_timestamp(86400)
# '1d 00:00:00'
diff --git a/wrolpi/files/downloader.py b/wrolpi/files/downloader.py
index 8c553e9a..367fd04b 100644
--- a/wrolpi/files/downloader.py
+++ b/wrolpi/files/downloader.py
@@ -1,4 +1,3 @@
-import asyncio
import pathlib
import traceback
from abc import ABC
diff --git a/wrolpi/files/ebooks.py b/wrolpi/files/ebooks.py
index dde3591a..0b76b153 100644
--- a/wrolpi/files/ebooks.py
+++ b/wrolpi/files/ebooks.py
@@ -4,11 +4,10 @@ import logging
import pathlib
from typing import List, Optional, Tuple
-import bs4
from sqlalchemy import Column, Integer, String, BigInteger, ForeignKey, or_, and_
from sqlalchemy.orm import relationship
-from wrolpi.common import ModelHelper, Base, register_modeler
+from wrolpi.common import ModelHelper, Base, register_modeler, get_html_soup
from wrolpi.db import get_db_session
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
from wrolpi.files.models import FileGroup
@@ -66,7 +65,7 @@ class EBookData:
def extract_text(html: str) -> str:
- soup = bs4.BeautifulSoup(html, features='html.parser')
+ soup = get_html_soup(html)
text = soup.get_text()
return text
diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py
index 04381cdd..ba8762b0 100644
--- a/wrolpi/files/lib.py
+++ b/wrolpi/files/lib.py
@@ -1,4 +1,5 @@
import asyncio
+import dataclasses
import datetime
import functools
import glob
@@ -412,14 +413,16 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
AND file_group.size = EXCLUDED.size
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
)
- RETURNING id, indexed
+ RETURNING id, file_group.indexed AS old_indexed, indexed
'''
curs.execute(stmt)
- need_index = [i[0] for i in curs.fetchall() if i[1] is False]
- if need_index:
- refresh_logger.info(f'Invalidated indexes of {len(need_index)} file groups near {str(chunk[0])}')
+ results = list(curs.fetchall())
+ # Count the files that used to be indexed, but need to be re-indexed.
+ invalidated_files = len([i['id'] for i in results if i['old_indexed'] and not i['indexed']])
+ if invalidated_files:
+ refresh_logger.info(f'Invalidated indexes of {invalidated_files} file groups near {chunk[0]}')
else:
- refresh_logger.debug(f'Upserted {len(chunk)} files near {str(chunk[0])}')
+ refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}')
if non_primary_files:
# New files may have been added which change what primary paths exist. Delete any file_groups which
@@ -456,9 +459,15 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
directories, files = partition(lambda i: i.is_dir(), exists)
while directories:
directory = directories.pop(0)
- new_files, new_directories = get_files_and_directories(directory)
+ try:
+ new_files, new_directories = get_files_and_directories(directory)
+ except FileNotFoundError as e:
+ # Directory may have been deleted during refresh.
+ logger.warning(f'Cannot refresh directory because it is missing: {directory}', exc_info=e)
+ continue
directories.extend(new_directories)
files.extend(new_files)
+ refresh_logger.debug(f'Discovered {len(new_files)} files in {directory}')
if len(files) >= 100:
# Wait until there are enough files to perform the upsert.
try:
@@ -475,6 +484,8 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
except Exception as e:
refresh_logger.error(f'Failed to upsert files', exc_info=e)
+ refresh_logger.info('Finished discovering files. Will now remove deleted files from DB...')
+
with get_db_curs(commit=True) as curs:
wheres = ''
if paths:
@@ -512,6 +523,9 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
if isinstance(paths, pathlib.Path):
paths = [paths, ]
+ idempotency = now()
+ REFRESH['idempotency'] = idempotency
+
refreshing_all_files = False
with flags.refreshing:
@@ -523,8 +537,6 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
if send_events:
Events.send_global_refresh_started()
- idempotency = now()
-
# Add all files in the media directory to the DB.
paths = paths or [get_media_directory()]
@@ -591,6 +603,7 @@ async def apply_indexers():
file_groups = session.query(FileGroup).filter(FileGroup.indexed != True).limit(20)
file_groups: List[FileGroup] = list(file_groups)
+ file_group = None
processed = 0
for file_group in file_groups:
processed += 1
@@ -606,7 +619,8 @@ async def apply_indexers():
# Sleep to catch cancel.
await asyncio.sleep(0)
- refresh_logger.debug(f'Indexed {processed} files')
+ if file_group:
+ refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}')
if processed < 20:
# Processed less than the limit, don't do the next query.
@@ -969,9 +983,52 @@ async def remove_file_group_tag(file_group_id: int, file_group_primary_path: str
file_group.remove_tag(tag, session)
-def get_refresh_progress():
- with get_db_curs() as curs:
- curs.execute('''
+@dataclasses.dataclass
+class RefreshProgress:
+ counted_files: int = 0
+ counting: bool = False
+ discovery: bool = False
+ indexed: int = 0
+ indexing: bool = False
+ modeled: int = 0
+ modeling: bool = False
+ cleanup: bool = False
+ refreshing: bool = False
+ total_file_groups: int = 0
+ unindexed: int = 0
+
+ def __json__(self):
+ d = dict(
+ counted_files=self.counted_files,
+ counting=self.counting,
+ discovery=self.discovery,
+ indexed=self.indexed,
+ indexing=self.indexing,
+ modeled=self.modeled,
+ modeling=self.modeling,
+ cleanup=self.cleanup,
+ refreshing=self.refreshing,
+ total_file_groups=self.total_file_groups,
+ unindexed=self.unindexed,
+ )
+ return d
+
+
+def get_refresh_progress() -> RefreshProgress:
+ idempotency = REFRESH.get('idempotency')
+ if idempotency:
+ stmt = '''
+ SELECT
+ -- Sum all the files in each FileGroup.
+ SUM(json_array_length(files)) FILTER (WHERE idempotency=%(idempotency)s) AS "total_file_groups",
+ COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency=%(idempotency)s) AS "indexed",
+ COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
+ COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency=%(idempotency)s) AS "modeled"
+ FROM file_group
+ '''
+ else:
+ # Idempotency has not yet been declared.
+ stmt = '''
SELECT
-- Sum all the files in each FileGroup.
SUM(json_array_length(files)) AS "total_file_groups",
@@ -979,25 +1036,28 @@ def get_refresh_progress():
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
FROM file_group
- ''')
+ '''
+
+ with get_db_curs() as curs:
+ curs.execute(stmt, dict(idempotency=idempotency))
results = dict(curs.fetchone())
# TODO counts are wrong if we are not refreshing all files.
- status = dict(
+ progress = RefreshProgress(
counted_files=REFRESH.get('counted_files', 0),
counting=flags.refresh_counting.is_set(),
discovery=flags.refresh_discovery.is_set(),
- indexed=results['indexed'],
+ indexed=int(results['indexed'] or 0),
indexing=flags.refresh_indexing.is_set(),
- modeled=results['modeled'],
+ modeled=int(results['modeled'] or 0),
modeling=flags.refresh_modeling.is_set(),
cleanup=flags.refresh_cleanup.is_set(),
refreshing=flags.refreshing.is_set(),
- total_file_groups=results['total_file_groups'],
- unindexed=results['unindexed'],
+ total_file_groups=int(results['total_file_groups'] or 0),
+ unindexed=int(results['unindexed'] or 0),
)
- return status
+ return progress
async def estimate_search(search_str: str, tag_names: List[str]):
diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py
index 66b700e1..e775cdda 100644
--- a/wrolpi/files/models.py
+++ b/wrolpi/files/models.py
@@ -65,17 +65,20 @@ class FileGroup(ModelHelper, Base):
__tablename__ = 'file_group'
id: int = Column(BigInteger, primary_key=True)
+ # the Path of the file that can be modeled or indexed.
+ author = Column(String) # name of the author, maybe even a URL
data = Column(FancyJSON) # populated by the modeler
files = Column(FancyJSON, nullable=False) # populated during discovery
- idempotency = Column(TZDateTime)
- indexed = Column(Boolean, default=lambda: False, nullable=False)
+ idempotency = Column(TZDateTime) # used to track which files need to be deleted during refresh
+ indexed = Column(Boolean, default=lambda: False, nullable=False) # wrolpi.files.lib.apply_indexers
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
model = Column(String) # "video", "archive", "ebook", etc.
- modification_datetime = Column(TZDateTime)
- # the Path of the file that can be modeled or indexed.
+ modification_datetime = Column(TZDateTime) # the modification date of the file on disk
primary_path: pathlib.Path = Column(MediaPathType, nullable=False, unique=True)
+ published_datetime = Column(TZDateTime) # the date the creator published this file
+ published_modified_datetime = Column(TZDateTime) # the date the publisher modified this file
size = Column(BigInteger, default=lambda: 0)
- title = Column(String)
+ title = Column(String) # user-displayable title
tag_files: InstrumentedList = relationship('TagFile', cascade='all')
@@ -114,6 +117,9 @@ class FileGroup(ModelHelper, Base):
'tags': tags,
'title': self.title,
'key': self.primary_path,
+ 'author': self.author,
+ 'published_datetime': self.published_datetime,
+ 'published_modified_datetime': self.published_modified_datetime,
}
return d
diff --git a/wrolpi/files/pdfs.py b/wrolpi/files/pdfs.py
index ccd723b3..930033b4 100644
--- a/wrolpi/files/pdfs.py
+++ b/wrolpi/files/pdfs.py
@@ -3,6 +3,9 @@ import dataclasses
import pathlib
from typing import Generator
+import pytz
+
+from wrolpi import dates
from wrolpi.common import register_modeler, logger, truncate_generator_bytes, truncate_object_bytes, \
split_lines_by_length
from wrolpi.db import get_db_session
@@ -24,31 +27,8 @@ __all__ = ['get_pdf_metadata', 'get_words', 'pdf_modeler']
class PDFMetadata:
title: str = None
author: str = None
- text: str = None
-
-
-def create_index(path):
- file_title = Indexer.get_title(path)
-
- reader = PdfReader(path)
-
- data = get_pdf_metadata(reader, path)
-
- if path.stat().st_size > FILE_MAX_PDF_SIZE:
- logger.warning(f'PDF too large to fully index: {path}')
- return data.title, data.author, file_title, None
-
- words = ''
- try:
- # PDFs are complex, don't fail to create title index if text extraction fails.
- words = '\n'.join(truncate_generator_bytes(get_words(reader, path), FILE_MAX_TEXT_SIZE))
- words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
- words = split_lines_by_length(words)
- except Exception as e:
- logger.error(f'Failed to index {path}', exc_info=e)
- if PYTEST:
- raise
- return data.title, data.author, file_title, words
+ published_datetime: str = None
+ modification_datetime: str = None
def get_pdf_metadata(reader: PdfReader, path: pathlib.Path) -> PDFMetadata:
@@ -60,8 +40,31 @@ def get_pdf_metadata(reader: PdfReader, path: pathlib.Path) -> PDFMetadata:
data.title = Indexer.get_title(path)
return data
- data.title = reader.metadata.title
- data.author = reader.metadata.author
+ if (title := reader.metadata.title) and title.lower() != 'unknown':
+ data.title = title
+ if (author := reader.metadata.author) and author.lower() != 'unknown':
+ data.author = author
+
+ if reader.metadata.creation_date_raw:
+ try:
+ str(reader.metadata.creation_date)
+ data.published_datetime = reader.metadata.creation_date
+ except ValueError:
+ # pypdf could not parse the date, lets give it a try...
+ data.published_datetime = dates.strpdate(reader.metadata.creation_date_raw)
+ if not data.published_datetime.tzinfo:
+ data.published_datetime = data.published_datetime.astimezone(pytz.UTC)
+
+ if reader.metadata.modification_date_raw:
+ try:
+ str(reader.metadata.modification_date)
+ data.modification_datetime = reader.metadata.modification_date
+ except ValueError:
+ # pypdf could not parse the date, lets give it a try...
+ data.modification_datetime = dates.strpdate(reader.metadata.modification_date_raw)
+ if not data.modification_datetime.tzinfo:
+ data.modification_datetime = data.modification_datetime.astimezone(pytz.UTC)
+
return data
@@ -117,12 +120,32 @@ async def pdf_modeler():
# Assume the first PDF is the only PDF.
pdf_file = pdf_files[0]['path']
try:
- title, author, file_title, contents = create_index(pdf_file)
- file_group.data = {'author': author}
- file_group.title = file_group.a_text = title
- file_group.b_text = author
- file_group.c_text = file_title
- file_group.d_text = contents
+ file_title = Indexer.get_title(pdf_file)
+
+ reader = PdfReader(pdf_file)
+
+ metadata = get_pdf_metadata(reader, pdf_file)
+
+ words = ''
+ if pdf_file.stat().st_size > FILE_MAX_PDF_SIZE:
+ logger.warning(f'PDF too large to fully index: {pdf_file}')
+ else:
+ try:
+ # PDFs are complex, don't fail to create title index if text extraction fails.
+ words = '\n'.join(truncate_generator_bytes(get_words(reader, pdf_file), FILE_MAX_TEXT_SIZE))
+ words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
+ words = split_lines_by_length(words)
+ except Exception as e:
+ logger.error(f'Failed to index {pdf_file}', exc_info=e)
+ if PYTEST:
+ raise
+
+ file_group.title = file_group.a_text = metadata.title or file_title
+ file_group.author = file_group.b_text = metadata.author
+ file_group.c_text = file_title # The name of the file may not match the title in the PDF metadata.
+ file_group.d_text = words or None
+ file_group.published_datetime = metadata.published_datetime
+ file_group.published_modified_datetime = metadata.modification_datetime
file_group.model = 'pdf'
except Exception as e:
logger.error(f'Failed to index PDF {pdf_file}', exc_info=e)
diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py
index 61394f5a..feb0ee96 100644
--- a/wrolpi/files/test/test_lib.py
+++ b/wrolpi/files/test/test_lib.py
@@ -13,7 +13,7 @@ import pytest
from PIL import Image
from wrolpi.common import timer
-from wrolpi.dates import now, from_timestamp
+from wrolpi.dates import now
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
from wrolpi.files import lib, indexers
from wrolpi.files.models import FileGroup
diff --git a/wrolpi/files/test/test_pdfs.py b/wrolpi/files/test/test_pdfs.py
index ca4608a5..c7171b27 100644
--- a/wrolpi/files/test/test_pdfs.py
+++ b/wrolpi/files/test/test_pdfs.py
@@ -13,8 +13,8 @@ async def test_pdf_title(test_session, example_pdf):
"""PDF modeler extracts a PDF's title and contents."""
await files_lib.refresh_files()
- with mock.patch('wrolpi.files.pdfs.create_index') as mock_create_index:
- mock_create_index.side_effect = Exception('Should not index twice')
+ with mock.patch('wrolpi.files.pdfs.get_pdf_metadata') as mock_get_pdf_metadata:
+ mock_get_pdf_metadata.side_effect = Exception('Should not index twice')
await files_lib.refresh_files()
pdf: FileGroup = test_session.query(FileGroup).one()
@@ -23,7 +23,11 @@ async def test_pdf_title(test_session, example_pdf):
assert pdf.b_text == 'roland'
assert pdf.c_text == 'pdf example pdf'
assert pdf.d_text and pdf.d_text.startswith('Page one\n') and len(pdf.d_text) == 467
- assert pdf.data and pdf.data.get('author') == 'roland'
+ # Metadata within the PDF.
+ assert pdf.author
+ assert pdf.published_datetime
+ # The date the file was modified.
+ assert pdf.modification_datetime
@pytest.mark.asyncio
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 7018b70b..615edc5c 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -17,13 +17,13 @@ from sanic_ext.extensions.openapi import openapi
from vininfo import Vin
from vininfo.details._base import VinDetails
-from wrolpi import admin, status, flags, schema
+from wrolpi import admin, status, flags, schema, dates
from wrolpi import tags
from wrolpi.admin import HotspotStatus
from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, LOG_LEVEL, \
set_global_log_level
-from wrolpi.dates import now, strptime
+from wrolpi.dates import now
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, APIError, HotspotError, InvalidDownload, \
HotspotPasswordTooShort, NativeOnly, InvalidConfig
@@ -411,7 +411,7 @@ async def get_statistics(_):
@validate(query=schema.EventsRequest)
async def feed(_: Request, query: schema.EventsRequest):
start = now()
- after = None if query.after == 'None' else strptime(query.after)
+ after = None if query.after == 'None' else dates.strpdate(query.after)
events = get_events(after)
return json_response(dict(events=events, now=start))
diff --git a/wrolpi/scrape_downloader.py b/wrolpi/scrape_downloader.py
index bd75f4c4..36c77b4e 100644
--- a/wrolpi/scrape_downloader.py
+++ b/wrolpi/scrape_downloader.py
@@ -2,9 +2,7 @@ import pathlib
from http import HTTPStatus
from urllib.parse import urlparse
-import bs4
-
-from wrolpi.common import aiohttp_session, logger
+from wrolpi.common import aiohttp_session, logger, get_html_soup
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.vars import DEFAULT_HTTP_HEADERS
@@ -90,7 +88,7 @@ class ScrapeHTMLDownloader(Downloader):
# Find all anchors, search for matching files, and more URLs to search.
try:
- soup = bs4.BeautifulSoup(content, 'html.parser')
+ soup = get_html_soup(content)
except Exception as e:
logger.error(f'Failed to parse HTML from {url}', exc_info=e)
continue
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index 3a2446c1..09f9f7bb 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -302,28 +302,6 @@ def test_escape_file_name(name, expected):
raise AssertionError(f'Escape of {repr(name)}: {repr(result)} != {repr(expected)}')
-@pytest.mark.parametrize(
- 'paths,suffix_groups,expected', [
- ([], [], ()),
- (['foo.mp4', 'foo.info.json'], [('.mp4',)], ('foo.mp4',)),
- (['foo.mp4', 'foo.info.json'], [('.mp4',), ('.info.json',)], ('foo.mp4', 'foo.info.json')),
- (['foo.mp4', 'foo.info.json'], [('.mp4',), ('.nope',), ('.info.json',)], ('foo.mp4', None, 'foo.info.json')),
- (['foo.mp4', 'foo.info.json', 'extra.txt'], [('.mp4',)], ('foo.mp4',)),
- (['foo.mp4'], [('.mp4', '.flv'), ('.info.json',)], ('foo.mp4', None)),
- (
- # Two files are matched to the closest suffix.
- ['foo.info.json', 'bar.json'],
- [('.info.json',), ('.json',)], # TODO longest suffix must be first
- ('foo.info.json', 'bar.json'),
- ),
- ]
-)
-def test_match_paths_to_suffixes(paths, suffix_groups, expected):
- paths = [pathlib.Path(i) for i in paths]
- expected = tuple(pathlib.Path(i) if i else None for i in expected)
- assert (i := common.match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}'
-
-
def test_truncate_object_bytes():
"""
Objects can be truncated (lists will be shortened) so they will fit in tsvector columns.
diff --git a/wrolpi/test/test_dates.py b/wrolpi/test/test_dates.py
index d919beb0..b5539ecc 100644
--- a/wrolpi/test/test_dates.py
+++ b/wrolpi/test/test_dates.py
@@ -1,7 +1,8 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
import pytest
import pytz
+from dateutil.tz import tzoffset
from sqlalchemy import Column, Integer
from wrolpi import dates
@@ -86,18 +87,54 @@ def test_timedelta_to_timestamp(td, expected):
@pytest.mark.parametrize('dt,expected', [
- ('2023-02-26T02:27:26.408944+00:00', datetime(2023, 2, 26, 2, 27, 26, 408944, tzinfo=pytz.UTC)),
- ('2023-02-26T02:27:26.408944 00:00', datetime(2023, 2, 26, 2, 27, 26, 408944, tzinfo=pytz.UTC)),
-])
-def test_strptime(dt, expected):
- assert dates.strptime(dt) == expected
-
-
-@pytest.mark.parametrize('dt,expected', [
- ('20010101', datetime(2001, 1, 1, tzinfo=pytz.UTC)),
- ('20050607', datetime(2005, 6, 7, tzinfo=pytz.UTC)),
- ('2006-07-08', datetime(2006, 7, 8, tzinfo=pytz.UTC)),
- ('2007-8-9', datetime(2007, 8, 9, tzinfo=pytz.UTC)),
+ ('20010101', datetime(2001, 1, 1)),
+ ('20050607', datetime(2005, 6, 7)),
+ ('2006-07-08', datetime(2006, 7, 8)),
+ ('2007-8-9', datetime(2007, 8, 9)),
+ ('2023-10-19T05:53:24+00:00', datetime(2023, 10, 19, 5, 53, 24, tzinfo=pytz.UTC)),
+ ('2022-09-27T00:40:19.000Z', datetime(2022, 9, 27, 0, 40, 19, tzinfo=pytz.UTC)),
+ ('06/17/2022', datetime(2022, 6, 17)),
+ ('14.3.2002', datetime(2002, 3, 14)),
+ ('2002.3.14', datetime(2002, 3, 14)),
+ ('2022/8/1', datetime(2022, 8, 1)),
+ ('2022-8-1', datetime(2022, 8, 1)),
+ ('2023-08-22T02:45:59+0000', datetime(2023, 8, 22, 2, 45, 59, tzinfo=pytz.UTC)),
+ ('2023-08-29T15:29-0400', datetime(2023, 8, 29, 15, 29, tzinfo=tzoffset(None, -14400))),
+ ('2023-08-29T15:29+0400', datetime(2023, 8, 29, 15, 29, tzinfo=tzoffset(None, 14400))),
+ ('2011-11-04', datetime(2011, 11, 4, 0, 0)),
+ ('20111104', datetime(2011, 11, 4, 0, 0)),
+ ('2011-11-04T00:05:23', datetime(2011, 11, 4, 0, 5, 23)),
+ ('2011-11-04T00:05:23Z', datetime(2011, 11, 4, 0, 5, 23, tzinfo=pytz.UTC)),
+ ('20111104T000523', datetime(2011, 11, 4, 0, 5, 23)),
+ ('2011-W01-2T00:05:23.283', datetime(2011, 1, 4, 0, 5, 23, 283000)),
+ ('2011-11-04 00:05:23.283', datetime(2011, 11, 4, 0, 5, 23, 283000)),
+ ('2011-11-04 00:05:23.283+00:00', datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=pytz.UTC)),
+ ('2011-11-04T00:05:23+04:00', datetime(2011, 11, 3, 20, 5, 23, tzinfo=pytz.UTC)),
+ ('2000', datetime(2000, 1, 1, tzinfo=pytz.UTC)),
+ ('Tuesday, October 19, 1999 3:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)),
+ ('Tue, October 19, 1999 3:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)),
+ ('Tue, October 19, 1999 03:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)),
+ # From SingleFile.
+ ('Fri Jun 17 2022 19:24:52', datetime(2022, 6, 17, 19, 24, 52)),
+ # PDFs are the Wild West...
+ ("D:20221226113758-07'00", datetime(2022, 12, 26, 11, 37, 58, tzinfo=timezone(timedelta(days=-1, seconds=61200)))),
+ ('D:20200205184724', datetime(2020, 2, 5, 18, 47, 24)),
+ ("D:20091019120104+", datetime(2009, 10, 19, 12, 1, 4)),
])
def test_strpdate(dt, expected):
assert dates.strpdate(dt) == expected
+
+
+def test_invalid_strpdate():
+ with pytest.raises(RuntimeError):
+ dates.strpdate('1')
+ with pytest.raises(RuntimeError):
+ dates.strpdate('asdf')
+ with pytest.raises(RuntimeError):
+ dates.strpdate('13/1/2008')
+
+ # Not a real date.
+ with pytest.raises(RuntimeError):
+ dates.strpdate('2001-2-31')
+ with pytest.raises(RuntimeError):
+ dates.strpdate('2001-2-30')
diff --git a/wrolpi/vars.py b/wrolpi/vars.py
index 1ac1dc32..98a9cadd 100644
--- a/wrolpi/vars.py
+++ b/wrolpi/vars.py
@@ -46,8 +46,6 @@ def truthy_arg(value: str) -> bool:
# Get the media directory from the environment.
DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')
MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY))
-if not MEDIA_DIRECTORY.is_dir() and not PYTEST:
- print(f'Media directory does not exist! {MEDIA_DIRECTORY}', file=sys.stderr)
CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config'
MODULES_DIR: Path = PROJECT_DIR / 'modules'