rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Adding FileGroup.published_date/author/etc. Creating script which converts a Singlefile to an Archive FileGroup files.

Browse source 
Parsing SingleFile datetime using dates.strpdate.

Adding timezone when no timezone info is available.

Installing python-dateutil to assist in handling diverse datetime strings.

Reworking refresh progress to use idempotency to display more accurate progress.  Only logging that files have been invalidated if they used to be indexed.

Clearing archives to force re-index.

Ignoring 'Unknown' author from PDF metadata. Using file_title when pdf metadata title is empty.
This commit is contained in:
Roland 2023-10-20 18:53:26 -06:00
parent 56efc03786
commit ae9d188391
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
40 changed files with 1023 additions and 305 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitmodules vendored
View file

@ -1,3 +1,3 @@
[submodule "docker/help/wrolpi-help"]
path = docker/help/wrolpi-help
url = https://github.com/lrnselfreliance/wrolpi-help.git
url = https://github.com/wrolpi/wrolpi-help.git

32
alembic/versions/4b54ad052e2e_reindex_archives.py Normal file
View file

@ -0,0 +1,32 @@
"""Reindex archives.
Revision ID: 4b54ad052e2e
Revises: 6b0b249a4f81
Create Date: 2023-10-20 13:28:55.517086
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '4b54ad052e2e'
down_revision = '6b0b249a4f81'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
# Archives must be re-indexed because of the previous revision.
session.execute("UPDATE file_group SET indexed=false WHERE model='archive'")
session.execute("DELETE FROM archive") # noqa
def downgrade():
pass

42
alembic/versions/6b0b249a4f81_archive_published_datetime.py Normal file
View file

@ -0,0 +1,42 @@
"""Archive published_datetime
Revision ID: 6b0b249a4f81
Revises: 4c8da5502c8a
Create Date: 2023-10-20 12:54:55.879180
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '6b0b249a4f81'
down_revision = '4c8da5502c8a'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE file_group ADD COLUMN author TEXT')
session.execute('ALTER TABLE file_group ADD COLUMN published_datetime TIMESTAMP WITH TIME ZONE')
session.execute('ALTER TABLE file_group ADD COLUMN published_modified_datetime TIMESTAMP WITH TIME ZONE')
session.execute('CREATE INDEX IF NOT EXISTS file_group_author_idx ON file_group(author)')
session.execute('CREATE INDEX IF NOT EXISTS file_group_published_datetime_idx ON file_group(published_datetime)')
session.execute('CREATE INDEX IF NOT EXISTS file_group_published_modified_datetime_idx'
' ON file_group(published_modified_datetime)')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP INDEX IF EXISTS file_group_author_idx')
session.execute('DROP INDEX IF EXISTS file_group_published_datetime_idx')
session.execute('DROP INDEX IF EXISTS file_group_published_modified_datetime_idx')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS author')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_datetime')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_modified_datetime')

29
app/src/components/Archive.js
View file

@ -6,6 +6,8 @@ import {
CardMeta,
Container,
Dropdown,
GridColumn,
GridRow,
Image,
Input,
PlaceholderHeader,
@ -18,7 +20,8 @@ import {
BackButton,
CardLink,
cardTitleWrapper,
encodeMediaPath, ErrorMessage,
encodeMediaPath,
ErrorMessage,
ExternalCardLink,
FileIcon,
findPosterPath,
@ -165,6 +168,13 @@ function ArchivePage() {
await fetchArchive();
}
const archivedDatetimeString = isoDatetimeToString(data.archive_datetime, true);
const publishedDatetimeString = archiveFile.published_datetime ? isoDatetimeToString(archiveFile.published_datetime, true)
: 'unknown';
const modifiedDatetimeString = archiveFile.published_modified_datetime
? isoDatetimeToString(archiveFile.published_modified_datetime, true)
: 'unknown';
const aboutPane = {
menuItem: 'About', render: () => <TabPane>
<Header as={'h3'}>Domain</Header>
@ -175,6 +185,9 @@ function ArchivePage() {
<Header as={'h3'}>URL</Header>
<p>{data.url ? <a href={data.url}>{data.url}</a> : 'N/A'}</p>
<Header as={'h3'}>Modified Date</Header>
<p>{modifiedDatetimeString}</p>
</TabPane>
};
@ -221,7 +234,19 @@ function ArchivePage() {
<ExternalCardLink to={singlefileUrl}>
<Header as='h2'>{textEllipsis(archiveFile.title || data.url)}</Header>
</ExternalCardLink>
<Header as='h3'>{isoDatetimeToString(data.archive_datetime, true)}</Header>
<Header as='h3'>Author: {archiveFile.author ? archiveFile.author : 'unknown'}</Header>
<Grid columns={2} stackable>
<GridRow>
<GridColumn>
<Header as='h4'>Published: {publishedDatetimeString}</Header>
</GridColumn>
<GridColumn>
<Header as='h4'>Archived: {archivedDatetimeString}</Header>
</GridColumn>
</GridRow>
</Grid>
{singlefileButton}
{updateButton}

5
app/src/components/Common.js
View file

@ -179,7 +179,10 @@ export function isoDatetimeToString(dt, time = false) {
let d = <React.Fragment/>;
if (dt && time) {
d = new Date(dt);
d = `${d.toDateString()} ${d.getHours()}:${d.getMinutes()}:${d.getSeconds()}`;
const hours = String(d.getHours()).padStart(2, '0');
const minutes = String(d.getMinutes()).padStart(2, '0');
const seconds = String(d.getSeconds()).padStart(2, '0');
d = `${d.toDateString()} ${hours}:${minutes}:${seconds}`;
} else if (dt) {
d = new Date(dt);
d = d.toDateString();

6
app/src/components/Files.js
View file

@ -137,11 +137,7 @@ function FileCard({file}) {
return <EbookCard key={file['primary_path']} file={file}/>;
}
const {data} = file;
let author;
if (data) {
author = data['author'];
}
const author = file.author;
const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
const color = mimetypeColor(file.mimetype);
const size = file.size !== null && file.size !== undefined ? humanFileSize(file.size) : null;

4
debian-live-config/config/package-lists/wrolpi.list.chroot
View file

@ -1,5 +1,6 @@
apache2
apache2-dev
apache2-doc
apt-transport-https
autoconf
build-essential
@ -17,8 +18,10 @@ fonts-noto-cjk
fonts-noto-hinted
fonts-noto-unhinted
gcc
gcc-doc
gdal-bin
git
git-doc
gnupg-agent
hostapd
htop
@ -64,6 +67,7 @@ osm2pgsql
osmium-tool
pkg-config
postgis
postgis-doc
postgresql-15
postgresql-15-postgis-3
postgresql-15-postgis-3-scripts

92
etc/raspberrypios/50x.html
View file

@ -1,3 +1,4 @@
<!DOCTYPE html>
<html lang="en">
<body>
@ -5,8 +6,8 @@
<p>nginx could not find WROLPi! Are your services running?</p>
<p><b>sudo systemctl status wrolpi-app</b></p>
<p><b>sudo systemctl status wrolpi-api</b></p>
<pre><b>sudo systemctl status wrolpi-app</b></pre>
<pre><b>sudo systemctl status wrolpi-api</b></pre>
<br/>
<p>If either of those services are not running, start all WROLPi services:</p>
@ -15,71 +16,72 @@
<br/>
<p>Or, start them individually:</p>
<p><b>sudo systemctl start wrolpi-app</b></p>
<p><b>sudo systemctl start wrolpi-api</b></p>
<p><b>sudo systemctl start wrolpi-kiwix</b></p>
<pre><b>sudo systemctl start wrolpi-app</b></pre>
<pre><b>sudo systemctl start wrolpi-api</b></pre>
<pre><b>sudo systemctl start wrolpi-kiwix</b></pre>
<br/>
<p>View the logs</p>
<p><b>journalctl -u wrolpi-app</b></p>
<p><b>journalctl -u wrolpi-api</b></p>
<p><b>journalctl -u wrolpi-kiwix</b></p>
<p>Watch the logs</p>
<p><b>journalctl -fu wrolpi-app</b></p>
<p><b>journalctl -fu wrolpi-api</b></p>
<p><b>journalctl -fu wrolpi-kiwix</b></p>
<p>View the logs:</p>
<pre><b>journalctl -u wrolpi-app</b></pre>
<pre><b>journalctl -u wrolpi-api</b></pre>
<pre><b>journalctl -u wrolpi-kiwix</b></pre>
<p>Watch the logs:</p>
<pre><b>journalctl -fu wrolpi-app</b></pre>
<pre><b>journalctl -fu wrolpi-api</b></pre>
<pre><b>journalctl -fu wrolpi-kiwix</b></pre>
<hr/>
<h2>Help script</h2>
<p>Run the provided help script which may narrow down the scope of the problem.</p>
<p><b>bash /opt/wrolpi/help.sh</b></p>
<pre><b>bash /opt/wrolpi/help.sh</b></pre>
<h2>Repair script</h2>
<p>Run the provided repair script which will attempt to automatically repair the WROLPi systems.</p>
<p><b>bash /opt/wrolpi/repair.sh</b></p>
<pre><b>bash /opt/wrolpi/repair.sh</b></pre>
<hr/>
<h2>Still not working?</h2>
<p>You can view the media directory by <a id="media_link" href="/media">navigating to /media</a></p>
<p>The API may be available <a id="api_link" href="http://127.0.0.1:8081">by navigating to port 8081</a></p>
<p>The map may be available <a id="map_link" href="http://127.0.0.1:8084">by navigating to port 8084</a></p>
<p>Kiwix may be available <a id="kiwix_link" href="http://127.0.0.1:8085">by navigating to port 8085</a></p>
<p>The API may be available <a id="api_link" href="#">by navigating to port 8081</a></p>
<hr/>
<p>The map may be available <a id="map_link" href="#">by navigating to port 8084</a></p>
<p>Kiwix may be available <a id="kiwix_link" href="#">by navigating to port 8085</a></p>
<h2>Third-Party Documentation</h2>
<p>WROLPi comes with official documentation from some third-party applications:</p>
<ul>
<li><a href="file:///usr/share/doc/python3-doc/html/index.html">Python 3</a></li>
<li><a href="file:///usr/share/doc/postgresql-doc-15/html/index.html">Postgresql</a></li>
<li><a href="file:///usr/share/doc/nodejs/api/index.html">NodeJS</a></li>
<li><a href="file:///usr/share/doc/apache2-doc/manual/en/index.html">Apache2</a></li>
<li><a href="file:///usr/share/doc/postgis-doc/postgis.html">Postgis</a></li>
<li><a href="file:///usr/share/doc/ffmpeg/manual/index.html">FFMPEG</a></li>
<li><a href="file:///usr/share/doc/gcc-10-doc/gcc.html">GCC</a></li>
<li><a href="file:///usr/share/doc/git-doc/git.html">Git</a></li>
</ul>
<script>
// Get the current hostname (probably an IP) link to my other ports above.
let hostname = window.location.hostname;
// Get the current hostname (probably an IP) link to the ports above.
const hostname = window.location.hostname;
let media_a = document.getElementById('media_link');
if (media_a) {
let media_url = `http://${hostname}:8080/media`;
media_a.setAttribute('href', media_url);
media_a.textContent = `by navigating to ${media_url}`;
function replaceHref(elm_id, url) {
const elm = document.getElementById(elm_id);
if (elm) {
elm.setAttribute('href', url);
elm.textContent = `by navigating to ${url}`;
}
}
let api_a = document.getElementById('api_link');
if (api_a) {
let api_url = `http://${hostname}:8081`;
api_a.setAttribute('href', api_url);
api_a.textContent = `by navigating to ${api_url}`;
}
let map_a = document.getElementById('map_link');
if (map_a) {
let map_url = `http://${hostname}:8084`;
map_a.setAttribute('href', map_url);
map_a.textContent = `by navigating to ${map_url}`;
}
let kiwix_a = document.getElementById('kiwix_link');
if (kiwix_a) {
let kiwix_url = `http://${hostname}:8085`;
kiwix_a.setAttribute('href', kiwix_url);
kiwix_a.textContent = `by navigating to ${kiwix_url}`;
if (hostname) {
replaceHref('media_link', `http://${hostname}:8080/media`);
replaceHref('api_link', `http://${hostname}:8081`);
replaceHref('map_link', `http://${hostname}:8084`);
replaceHref('kiwix_link', `http://${hostname}:8085`);
} else {
console.error('Refusing to replace services links because window.location.hostname is empty');
}
</script>

14
modules/archive/__init__.py
View file

@ -238,20 +238,21 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
return archive
except Exception as e:
logger.error(f'Failed to model Archive {file_group}', exc_info=e)
if PYTEST:
raise
logger.error('Failed to model Archive', exc_info=e)
raise InvalidArchive(f'Failed to model Archive {file_group}') from e
@register_modeler
async def archive_modeler():
"""Searches DB for FileGroups that contain an HTML file. If the HTML file is a SingleFile, we model it as an
Archive."""
invalid_archives = set()
while True:
with get_db_session(commit=True) as session:
results = session.query(FileGroup, Archive) \
.filter(
# Get all groups that contain a PDF that have not been indexed.
# Get all groups that contain an HTML file that have not been indexed.
FileGroup.indexed != True,
FileGroup.mimetype == 'text/html',
).filter(not_(FileGroup.id.in_(list(invalid_archives)))) \
@ -262,7 +263,8 @@ async def archive_modeler():
for file_group, archive in results:
processed += 1
with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}', logger__=logger):
with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}',
logger__=logger):
if archive:
archive: Archive
try:
@ -276,7 +278,7 @@ async def archive_modeler():
try:
model_archive(file_group, session=session)
except InvalidArchive:
# May not be a real Singlefile archive.
# It was not a real Archive. Many HTML files will not be an Archive.
pass
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.

3
modules/archive/conftest.py
View file

@ -76,7 +76,8 @@ def archive_factory(test_session, archive_directory, make_files_structure):
archive = Archive.from_paths(test_session, *files)
archive.url = url
archive.title = title
archive.archive_datetime = next(now)
archive.archive_datetime = archive.file_group.published_datetime = next(now)
archive.file_group.modification_datetime = next(now)
archive.domain = domain
archive.validate()

188
modules/archive/lib.py
View file

@ -5,19 +5,22 @@ import pathlib
import re
from dataclasses import dataclass
from datetime import datetime
from typing import Optional, Tuple, List
from typing import Optional, Tuple, List, Union
from urllib.parse import urlparse
from bs4 import BeautifulSoup
import pytz
from sqlalchemy import asc
from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from wrolpi import dates
from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \
format_html_string, split_lines_by_length
format_html_string, split_lines_by_length, get_html_soup
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy
from wrolpi.tags import tag_names_to_file_group_sub_select
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@ -220,13 +223,157 @@ def get_title_from_html(html: str, url: str = None) -> str:
"""
Try and get the title from the
"""
soup = BeautifulSoup(html, features='html.parser')
soup = get_html_soup(html)
try:
return soup.title.string
except Exception: # noqa
logger.debug(f'Unable to extract title {url}')
SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n')
def get_url_from_singlefile(html: bytes) -> str:
"""Extract URL from SingleFile contents."""
html = html.decode()
if SINGLEFILE_HEADER not in html:
raise RuntimeError(f'Not a singlefile!')
url, = SINGLEFILE_URL_EXTRACTOR.findall(html)
url = url.strip()
if not url.startswith('http'):
raise RuntimeError('URL did not start with http')
# Verify that the URL can be parsed.
urlparse(url)
return url
@dataclass
class ArticleMetadata:
title: str = None
published_datetime: datetime = None
modified_datetime: datetime = None
description: str = None
author: str = None
def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True) -> ArticleMetadata:
"""
Read the data from the <meta> tags, extract any data relevant to the article. This function also reads the
<script type="application/ld+json"> data.
"""
metadata = ArticleMetadata()
soup = get_html_soup(html)
def get_meta_by_property(prop: str):
return soup.find('meta', attrs={'property': prop})
# <meta content="2023-10-18T04:52:23+00:00" property="article:published_time"/>
if meta_published_time := get_meta_by_property('article:published_time'):
metadata.published_datetime = dates.strpdate(meta_published_time.attrs['content'])
logger.debug(f'Found meta published_time: {meta_published_time}')
# <meta name="article.published" content="2023-04-04T21:52:00.000Z">
if meta_article_published := soup.find('meta', attrs={'name': 'article.published'}):
metadata.published_datetime = metadata.published_datetime or dates.strpdate(
meta_article_published.attrs['content'])
# <meta itemprop="datePublished" content="2023-04-04T21:52:00.000Z">
if meta_item_published := soup.find('meta', attrs={'itemprop': 'datePublished'}):
metadata.published_datetime = metadata.published_datetime or dates.strpdate(
meta_item_published.attrs['content'])
# <time itemprop="datePublished" datetime="2023-08-25">
if time := soup.find('time', attrs={'itemprop': 'datePublished'}):
metadata.published_datetime = metadata.published_datetime or dates.strpdate(time.attrs['datetime'])
logger.debug(f'Found time published_time: {time}')
# <abbr class="published" itemprop="datePublished" title="2022-03-17T03:00:00-07:00">March 17, 2022</abbr>
if abbr := soup.find('abbr', attrs={'itemprop': 'datePublished'}):
metadata.published_datetime = metadata.published_datetime or dates.strpdate(abbr.attrs['title'])
# <meta content="2023-10-19T05:53:24+00:00" property="article:modified_time"/>
if meta_modified_time := get_meta_by_property('article:modified_time'):
metadata.modified_datetime = dates.strpdate(meta_modified_time.attrs['content'])
logger.debug(f'Found meta modified_time: {meta_modified_time}')
# <meta name="article.updated" content="2023-04-04T21:52:00.000Z">
if meta_article_updated := soup.find('meta', attrs={'name': 'article.updated'}):
metadata.modified_datetime = metadata.modified_datetime or dates.strpdate(meta_article_updated.attrs['content'])
# <meta content="The Title" property="og:title"/>
if meta_title := get_meta_by_property('og:title'):
metadata.title = meta_title.attrs['content']
logger.debug(f'Found meta title: {meta_title}')
# <meta name="author" content="Author Name"/>
if meta_author := soup.find('meta', attrs={'name': 'author'}):
metadata.author = meta_author.attrs['content']
logger.debug(f'Found meta author: {meta_author}')
# <meta content="Billy" property="article:author"/>
if meta_property_author := get_meta_by_property('article:author'):
metadata.author = metadata.author or meta_property_author.attrs['content']
# <a href="https://example.com" rel="author">
if link_author := soup.find('a', attrs={'rel': 'author'}):
metadata.author = metadata.author or link_author.text.strip()
logger.debug(f'Found link author: {link_author}')
# <script class="sf-hidden" type="application/ld+json">
if (ld_script := soup.find('script', attrs={'type': 'application/ld+json'})) and ld_script.contents:
try:
schema = json.loads(ld_script.text)
except json.decoder.JSONDecodeError:
# Was not valid JSON.
schema = None
if isinstance(schema, dict) and schema.get('@context') in ('https://schema.org', 'http://schema.org'):
# Found https://schema.org/
if headline := schema.get('headline'):
metadata.title = metadata.title or schema.get('headline')
logger.debug(f'Found schema headline: {headline}')
if datePublished := schema.get('datePublished'):
try:
metadata.published_datetime = metadata.published_datetime or dates.strpdate(datePublished)
logger.debug(f'Found schema published_datetime: {datePublished}')
except RuntimeError as e:
# Invalid date, ignore.
logger.error('Invalid datetime', exc_info=e)
if PYTEST:
raise
if dateModified := schema.get('dateModified'):
try:
metadata.modified_datetime = metadata.modified_datetime or dates.strpdate(dateModified)
logger.debug(f'Found schema modified_datetime: {dateModified}')
except RuntimeError as e:
# Invalid date, ignore.
logger.error('Invalid datetime', exc_info=e)
if PYTEST:
raise
if description := schema.get('description'):
metadata.description = description
logger.debug(f'Found schema description: {description}')
if author := schema.get('author'):
if isinstance(author, list) and len(author) >= 1:
# Use the first Author.
author = author[0]
if isinstance(author, dict):
author = author.get('name') or author
if isinstance(author, str):
metadata.author = author
logger.debug(f'Found schema author: {author}')
else:
logger.warning(f'Unable to parse author schema: {author}')
# Assume UTC if no timezone.
if metadata.published_datetime and not metadata.published_datetime.tzinfo and assume_utc:
logger.debug(f'Assuming UTC for {metadata.published_datetime=}')
metadata.published_datetime = metadata.published_datetime.replace(tzinfo=pytz.UTC)
if metadata.modified_datetime and not metadata.modified_datetime.tzinfo and assume_utc:
logger.debug(f'Assuming UTC for {metadata.modified_datetime=}')
metadata.modified_datetime = metadata.modified_datetime.replace(tzinfo=pytz.UTC)
return metadata
@optional_session
def get_archive(session, archive_id: int) -> Archive:
"""Get an Archive."""
@ -292,22 +439,21 @@ def get_domains():
ARCHIVE_ORDERS = {
'date': 'a.archive_datetime ASC',
'-date': 'a.archive_datetime DESC NULLS LAST',
'rank': '2 DESC, a.archive_datetime DESC',
'-rank': '2 ASC, a.archive_datetime ASC',
'date': (date := 'fg.published_datetime ASC, a.archive_datetime ASC'),
'-date': (_date := 'fg.published_datetime DESC NULLS LAST, a.archive_datetime DESC NULLS LAST'),
'rank': f'2 DESC, {_date}',
'-rank': f'2 ASC, {date}',
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
'-size': 'fg.size DESC NULLS LAST, LOWER(fg.primary_path) DESC',
}
ORDER_GROUP_BYS = {
'date': 'a.archive_datetime',
'-date': 'a.archive_datetime',
'rank': 'a.archive_datetime',
'-rank': 'a.archive_datetime',
'size': 'fg.size, fg.primary_path',
'-size': 'fg.size, fg.primary_path',
'date': 'fg.published_datetime, a.archive_datetime',
'-date': 'fg.published_datetime, a.archive_datetime',
'rank': 'fg.published_datetime, a.archive_datetime',
'-rank': 'fg.published_datetime, a.archive_datetime',
'size': 'fg.size, fg.published_datetime, fg.primary_path',
'-size': 'fg.size, fg.published_datetime, fg.primary_path',
}
FILE_ORDERS = ['size', '-size']
def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str],
@ -315,11 +461,10 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
-> Tuple[List[dict], int]:
# Always filter FileGroups to Archives.
wheres = []
joins = []
group_by = 'a.file_group_id'
group_by = 'fg.published_datetime, a.archive_datetime, a.file_group_id'
params = dict(search_str=search_str, offset=int(offset), limit=int(limit))
order_by = 'id DESC'
order_by = ARCHIVE_ORDERS['-date']
select_columns = ''
if search_str:
@ -328,15 +473,12 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
' COUNT(*) OVER() AS total'
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
params['search_str'] = search_str
joins = ['LEFT JOIN file_group fg ON fg.id = a.file_group_id']
group_by = f'{group_by}, rank'
if order:
try:
order_by = ARCHIVE_ORDERS[order]
group_by = f'{group_by}, {ORDER_GROUP_BYS[order]}'
if order in FILE_ORDERS:
joins = ['LEFT JOIN file_group fg ON fg.id = a.file_group_id']
except KeyError:
raise InvalidOrderBy(f'Invalid order by: {order}')
@ -345,7 +487,6 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
tags_stmt, params_ = tag_names_to_file_group_sub_select(tag_names)
params.update(params_)
wheres.append(f'fg.id = ANY({tags_stmt})')
joins = ['LEFT JOIN file_group fg ON fg.id = a.file_group_id']
if search_str and headline:
headline = ''',
@ -364,7 +505,6 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
select_columns = f", {select_columns}" if select_columns else ""
wheres = '\n AND '.join(wheres)
where = f'WHERE\n{wheres}' if wheres else ''
join = '\n'.join(joins)
stmt = f'''
SELECT
a.file_group_id AS id, -- always get `file_group.id` for `handle_file_group_search_results`
@ -372,7 +512,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
{select_columns}
{headline}
FROM archive a
{join}
LEFT JOIN file_group fg ON fg.id = a.file_group_id
{where}
GROUP BY {group_by}
ORDER BY {order_by}

33
modules/archive/models.py
View file

@ -1,4 +1,3 @@
import datetime
import json
import pathlib
import re
@ -9,6 +8,7 @@ from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
from sqlalchemy.orm import relationship, Session, validates
from sqlalchemy.orm.collections import InstrumentedList
from wrolpi import dates
from wrolpi.common import ModelHelper, Base, logger
from wrolpi.dates import TZDateTime
from wrolpi.db import optional_session
@ -170,7 +170,7 @@ class Archive(Base, ModelHelper):
if screenshot_file := self.screenshot_file:
return screenshot_file['path']
def read_readability_data(self):
def apply_readability_data(self):
"""Read the Readability JSON file, apply its contents to this record."""
readability_json_path = self.readability_json_path
if not readability_json_path:
@ -195,7 +195,7 @@ class Archive(Base, ModelHelper):
self.url = url or self.url
self.file_group.title = title or self.file_group.title
def read_singlefile_data(self):
def apply_singlefile_data(self):
"""Read the start of the singlefile (if any) and extract any Archive information."""
path = self.singlefile_path
if not path:
@ -225,10 +225,7 @@ class Archive(Base, ModelHelper):
dt = match[0].strip()
dt = ' '.join(dt.split(' ')[:5])
# SingleFile uses GMT.
dt = datetime.datetime.strptime(
dt,
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
).replace(tzinfo=pytz.timezone('GMT'))
dt = dates.strpdate(dt).replace(tzinfo=pytz.timezone('GMT'))
self.archive_datetime = dt
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
@ -251,13 +248,31 @@ class Archive(Base, ModelHelper):
if self.singlefile_path and not self.file_group.title:
self.file_group.title = get_title_from_html(self.singlefile_path.read_text())
def apply_metadata(self):
"""Read and apply <meta> (and more) data from the Singlefile HTML."""
from modules.archive import lib
contents = self.singlefile_path.read_bytes()
metadata = lib.parse_article_html_metadata(contents)
if metadata.author:
self.file_group.author = metadata.author
if metadata.title:
self.file_group.title = metadata.title
if metadata.description:
self.file_group.b_text = metadata.description
if metadata.published_datetime:
self.file_group.published_datetime = metadata.published_datetime
if metadata.modified_datetime:
self.file_group.published_modified_datetime = metadata.modified_datetime
def validate(self):
"""Fill in any missing data about this Archive from its files."""
try:
self.read_readability_data()
self.read_singlefile_data()
self.apply_readability_data()
self.apply_singlefile_data()
self.apply_domain()
self.apply_singlefile_title()
self.apply_metadata()
except Exception as e:
logger.warning(f'Unable to validate {self}', exc_info=e)
if PYTEST:

174
modules/archive/test/test_lib.py
View file

@ -5,7 +5,9 @@ from datetime import datetime, timedelta
from http import HTTPStatus
import pytest
import pytz
from PIL import Image
from pytz import utc
from modules.archive import lib
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, model_archive_result, \
@ -63,7 +65,7 @@ async def test_dict(test_session):
@pytest.mark.asyncio
async def test_relationships(test_session, test_directory, example_singlefile):
async def test_relationships(test_session, example_singlefile):
with get_db_session(commit=True) as session:
url = 'https://wrolpi.org:443'
domain = get_or_create_domain(session, url)
@ -459,7 +461,8 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b
'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.png'
def test_refresh_archives(test_session, test_directory, test_client, make_files_structure):
@pytest.mark.asyncio
async def test_refresh_archives(test_session, test_directory, test_async_client, make_files_structure):
"""Archives can be found and put in the database. A single Archive will have multiple files."""
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
@ -484,7 +487,7 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
})
# The single archive is found.
test_client.post('/api/files/refresh')
await test_async_client.post('/api/files/refresh')
assert test_session.query(Archive).count() == 1
# Cause a re-index of the archive.
@ -493,7 +496,7 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
test_session.commit()
# Running the refresh does not result in a new archive.
test_client.post('/api/files/refresh')
await test_async_client.post('/api/files/refresh')
assert test_session.query(Archive).count() == 1
# Archives file format was changed, lets check the new formats are found.
@ -501,13 +504,13 @@ def test_refresh_archives(test_session, test_directory, test_client, make_files_
'archive/example.com/2021-10-05-16-20-11_The Title.html': '<html></html>',
'archive/example.com/2021-10-05-16-20-11_The Title.readability.json': '{"url": "https://example.com"}',
})
test_client.post('/api/files/refresh')
await test_async_client.post('/api/files/refresh')
# The old formatted archive above is renamed.
assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file()
assert test_session.query(Archive).count() == 2
content = json.dumps({'search_str': 'text', 'model': 'archive'})
request, response = test_client.post('/api/files/search', content=content)
request, response = await test_async_client.post('/api/files/search', content=content)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], 'No files matched "text"'
assert response.json['file_groups'][0]['model'] == 'archive', 'Returned file was not an archive'
@ -528,7 +531,14 @@ async def test_refresh_archives_index(test_session, make_files_structure):
Page saved with SingleFile
url: https://example.com
saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">'''
--><head><meta charset="utf-8">
<script class="sf-hidden" type="application/ld+json">
{"@context":"http://schema.org", "@type":"NewsArticle", "headline":"The headline",
"datePublished":"2022-09-27T00:40:19.000Z", "dateModified":"2022-09-27T13:43:47.971Z",
"author":{"@type":"Person", "name":"AUTHOR NAME", "jobTitle":""},
"description": "The article description"}
</script>
</html>'''
singlefile, *_ = make_files_structure({
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
@ -543,8 +553,14 @@ async def test_refresh_archives_index(test_session, make_files_structure):
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
assert archive.file_group.a_text == 'the title'
assert archive.file_group.d_text == 'article text contents'
assert archive.file_group.author == 'AUTHOR NAME'
assert archive.file_group.published_datetime
assert archive.file_group.published_modified_datetime
assert archive.file_group.a_text == 'the title' # from readability json
assert archive.file_group.b_text == 'The article description' # From application/ld+json
assert archive.file_group.d_text == 'article text contents' # from readability txt
assert archive.file_group.indexed is True
@ -651,3 +667,143 @@ def test_archive_history(test_session, archive_factory):
assert not archive3.history, 'archive3 has no history'
assert not archive4.history, 'archive4 has no history'
assert not archive5.history, 'archive5 has no history'
SINGLEFILE_1 = b'''
<!DOCTYPE html>
<html lang="en-US">
</html>
'''
SINGLEFILE_2 = b'''
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="utf-8"/>
<title>This is not a meta title and should be ignored</title>
<meta content="en_US" property="og:locale"/>
<meta content="article" property="og:type"/>
<meta content="The Title" property="og:title"/>
<meta name="author" content="Author Name"/>
<meta content="2023-10-18T04:52:23+00:00" property="article:published_time"/>
<meta content="2023-10-19T05:53:24+00:00" property="article:modified_time"/>
</head>
</html>
'''
SINGLEFILE_3 = b'''
<!DOCTYPE html>
<html lang="en-US">
<title>This is not a meta title and should be ignored</title>
<script class="sf-hidden" type="application/ld+json">
{"@context":"http://schema.org", "@type":"NewsArticle", "headline":"The headline",
"datePublished":"2022-09-27T00:40:19.000Z", "dateModified":"2022-09-27T13:43:47.971Z",
"author":{"@type":"Person", "name":"BOBBY", "jobTitle":""},
"creator":{"@type":"Person", "name":"OTHER BOBBY", "jobTitle":""},
"description": "The article description"}
</script>
</html>'''
SINGLEFILE_4 = b'''
<!DOCTYPE html>
<html lang="en-US">
<title>This is not a meta title and should be ignored</title>
<a href="https://example.com" rel="author">
Link Author
</a>
</html>'''
SINGLEFILE_5 = b'''
<!DOCTYPE html>
<html lang="en-US">
<title>This is not a meta title and should be ignored</title>
<a class='g-profile' href='https://example.com' rel='author' title='author profile'>
<span itemprop='name'>The Span Author</span>
</a>
<time class="post-date updated" itemprop="datePublished" datetime="2023-08-25"></time>
</html>
'''
def test_parse_article_html_metadata():
"""Singlefiles may contain metadata tags."""
# Metadata can be empty.
metadata = lib.parse_article_html_metadata(SINGLEFILE_1)
assert metadata.title is None
assert metadata.published_datetime is None
assert metadata.modified_datetime is None
assert metadata.description is None
assert metadata.author is None
# Data from <meta> elements.
metadata = lib.parse_article_html_metadata(SINGLEFILE_2)
assert metadata.title == 'The Title'
assert metadata.published_datetime == datetime(2023, 10, 18, 4, 52, 23, tzinfo=utc)
assert metadata.modified_datetime == datetime(2023, 10, 19, 5, 53, 24, tzinfo=utc)
assert metadata.description is None
assert metadata.author == 'Author Name'
# Data from Article structured data
# https://developers.google.com/search/docs/appearance/structured-data/article
metadata = lib.parse_article_html_metadata(SINGLEFILE_3)
assert metadata.title == 'The headline'
assert metadata.published_datetime == datetime(2022, 9, 27, 0, 40, 19, tzinfo=utc)
assert metadata.modified_datetime == datetime(2022, 9, 27, 13, 43, 47, 971000, tzinfo=utc)
assert metadata.description == 'The article description'
assert metadata.author == 'BOBBY'
# Data from author anchor element.
metadata = lib.parse_article_html_metadata(SINGLEFILE_4)
assert metadata.author == 'Link Author'
# author anchor may be a list.
metadata = lib.parse_article_html_metadata(SINGLEFILE_5)
assert metadata.author == 'The Span Author'
assert metadata.published_datetime == datetime(2023, 8, 25, tzinfo=pytz.UTC)
@pytest.mark.parametrize(
'html,expected',
[
('<meta content="2023-10-18T04:52:23+00:00" property="article:published_time"/>',
datetime(2023, 10, 18, 4, 52, 23, tzinfo=utc)),
('<abbr class="published" itemprop="datePublished" title="2022-03-17T03:00:00-07:00">March 17, 2022</abbr>',
datetime(2022, 3, 17, 10, 0, tzinfo=pytz.UTC)),
('<meta name="article.published" content="2023-04-04T21:52:00.000Z">',
datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)),
('<meta itemprop="datePublished" content="2023-04-04T21:52:00.000Z">',
datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)),
(
'<abbr class="published" itemprop="datePublished" title="2013-03-05T00:12:00-06:00">3/05/2013 12:12:00 AM</abbr>',
datetime(2013, 3, 5, 6, 12, tzinfo=pytz.UTC)),
('<time class="post-date updated" itemprop="datePublished" datetime="2023-08-25"></time>',
datetime(2023, 8, 25, tzinfo=pytz.UTC))
])
def test_published_datetime(html, expected):
assert lib.parse_article_html_metadata(html).published_datetime == expected
@pytest.mark.parametrize(
'html,expected',
[
('<meta name="article.updated" content="2023-04-04T21:52:00.000Z">',
datetime(2023, 4, 4, 21, 52, tzinfo=pytz.UTC)),
('<meta content="2023-10-19T05:53:24+00:00" property="article:modified_time"/>',
datetime(2023, 10, 19, 5, 53, 24, tzinfo=utc)),
(
'<script class="sf-hidden" type="application/ld+json">{"@context":"http://schema.org","dateModified":"2022-09-27T13:43:47.971Z"}</script>',
datetime(2022, 9, 27, 13, 43, 47, 971000, tzinfo=utc)),
])
def test_modified_datetime(html, expected):
assert lib.parse_article_html_metadata(html).modified_datetime == expected
@pytest.mark.parametrize(
'html,expected',
[
('<a href="#" rel="author"><span itemprop="name">Linda</span></a></span>', 'Linda'),
('<meta content="Billy" property="article:author"/>', 'Billy'),
]
)
def test_author(html, expected):
assert lib.parse_article_html_metadata(html).author == expected

30
modules/inventory/common.py
View file

@ -9,8 +9,8 @@ from wrolpi import before_startup
from wrolpi.common import logger, Base, ConfigFile
from wrolpi.db import get_db_session
from .errors import NoInventories, InventoriesVersionMismatch
from .inventory import unit_registry, get_items, get_inventories, increment_inventories_version, \
get_inventories_version
from .inventory import get_items, get_inventories, increment_inventories_version, \
get_inventories_version, get_unit_registry
from .models import Inventory, Item
MY_DIR: Path = Path(__file__).parent
@ -23,6 +23,8 @@ def sum_by_key(items: List[Base], key: callable):
Sum the total size of each item by the provided key function. Returns a dict containing the key, and the total_size
for that key. Combine total of like units. This means ounces and pounds will be in the same total.
"""
unit_registry = get_unit_registry()
summed = dict()
for item in items:
k = key(item)
@ -61,17 +63,28 @@ def get_inventory_by_keys(keys: Tuple, inventory_id: int):
get_inventory_by_category = partial(get_inventory_by_keys, ('category',))
get_inventory_by_subcategory = partial(get_inventory_by_keys, ('category', 'subcategory'))
get_inventory_by_name = partial(get_inventory_by_keys, ('brand', 'name'))
INVENTORY_UNITS = {
('ounce', 1): (16, unit_registry.pound),
('pound', 1): (2000, unit_registry.ton),
}
INVENTORY_UNITS: dict = None
UNIT_PRECISION = 5
def compact_unit(quantity: unit_registry.Quantity) -> unit_registry.Quantity:
def initialize_inventory_units() -> dict:
global INVENTORY_UNITS
if INVENTORY_UNITS:
return INVENTORY_UNITS
unit_registry = get_unit_registry()
INVENTORY_UNITS = {
('ounce', 1): (16, unit_registry.pound),
('pound', 1): (2000, unit_registry.ton),
}
def compact_unit(quantity: Quantity) -> Quantity:
"""
Convert a Quantity to it's more readable format. Such as 2000 pounds to 1 ton.
"""
initialize_inventory_units()
number, (units,) = quantity.to_tuple()
next_unit = INVENTORY_UNITS.get(units)
if not next_unit:
@ -88,7 +101,8 @@ def compact_unit(quantity: unit_registry.Quantity) -> unit_registry.Quantity:
return quantity
def quantity_to_tuple(quantity: unit_registry.Quantity) -> Tuple[Decimal, Quantity]:
def quantity_to_tuple(quantity: Quantity) -> Tuple[Decimal, Quantity]:
unit_registry = get_unit_registry()
decimal, (units,) = quantity.to_tuple()
unit, _ = units
quantity, unit = round(decimal, UNIT_PRECISION), unit_registry(unit)

33
modules/inventory/inventory.py
View file

@ -15,7 +15,38 @@ from .models import Inventory, Item, InventoriesVersion
logger = logger.getChild(__name__)
unit_registry = pint.UnitRegistry()
__all__ = [
'DEFAULT_CATEGORIES',
'DEFAULT_INVENTORIES',
'delete_inventory',
'delete_items',
'get_brands',
'get_categories',
'get_inventories',
'get_inventories_version',
'get_items',
'get_next_inventories_version',
'get_unit_registry',
'increment_inventories_version',
'logger',
'save_inventory',
'save_item',
'sort_categories',
'update_inventory',
'update_item',
]
UNIT_REGISTRY: pint.UnitRegistry = None
def get_unit_registry() -> pint.UnitRegistry:
global UNIT_REGISTRY
if UNIT_REGISTRY:
return UNIT_REGISTRY
UNIT_REGISTRY = pint.UnitRegistry()
return UNIT_REGISTRY
DEFAULT_CATEGORIES = [
('salt', 'cooking ingredients'),

19
modules/inventory/test/test_inventory.py
View file

@ -15,11 +15,12 @@ from .. import init
from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \
compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file, get_inventories_config
from ..errors import NoInventories, InventoriesVersionMismatch
from ..inventory import unit_registry, \
get_inventories, save_inventory, update_inventory, \
delete_inventory, get_categories
from ..inventory import get_inventories, save_inventory, update_inventory, \
delete_inventory, get_categories, get_unit_registry
from ..models import Item, Inventory
UNIT_REGISTRY = get_unit_registry()
TEST_ITEMS_COLUMNS = (
'inventory_id',
'brand',
@ -199,11 +200,11 @@ def quantity_to_string(quantity: Quantity) -> str:
@pytest.mark.parametrize(
'quantity,expected',
[
(Decimal(5) * unit_registry.ounce, '5 ounce'),
(Decimal(16) * unit_registry.ounce, '1 pound'),
(Decimal(500) * unit_registry.pound, '500 pound'),
(Decimal(2000) * unit_registry.pound, '1 ton'),
(Decimal(128000) * unit_registry.ounce, '4 ton'),
(Decimal(5) * UNIT_REGISTRY.ounce, '5 ounce'),
(Decimal(16) * UNIT_REGISTRY.ounce, '1 pound'),
(Decimal(500) * UNIT_REGISTRY.pound, '500 pound'),
(Decimal(2000) * UNIT_REGISTRY.pound, '1 ton'),
(Decimal(128000) * UNIT_REGISTRY.ounce, '4 ton'),
]
)
def test_compact_unit(quantity, expected):
@ -211,7 +212,7 @@ def test_compact_unit(quantity, expected):
assert quantity_to_string(compact_unit(quantity)) == expected
pound, oz, gram, gallon = unit_registry('pound'), unit_registry('oz'), unit_registry('gram'), unit_registry('gallon')
pound, oz, gram, gallon = UNIT_REGISTRY('pound'), UNIT_REGISTRY('oz'), UNIT_REGISTRY('gram'), UNIT_REGISTRY('gallon')
pound, oz, gram, gallon = pound.units, oz.units, gram.units, gallon.units
mass = pound.dimensionality
length = gallon.dimensionality

5
modules/videos/lib.py
View file

@ -6,6 +6,7 @@ import re
import warnings
from typing import Tuple, Optional, Generator
import pytz
from sqlalchemy import or_
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
@ -102,6 +103,8 @@ def validate_video(video: Video, channel_generate_poster: bool):
upload_date = upload_date or video.upload_date
if upload_date and isinstance(upload_date, str):
upload_date = dates.strpdate(upload_date)
if not upload_date.tzinfo:
upload_date = upload_date.astimezone(pytz.UTC)
video.file_group.title = video.file_group.title or html.unescape(title)
video.upload_date = upload_date
video.source_id = video.source_id or source_id
@ -120,7 +123,7 @@ def validate_video(video: Video, channel_generate_poster: bool):
if video.ffprobe_json:
if duration := video.ffprobe_json['format'].get('duration'):
video.duration = float(duration)
elif video_streams := video.get_streams_by_codec_type('video'):
elif (video_streams := video.get_streams_by_codec_type('video')) and 'duration' in video_streams[0]:
video.duration = float(video_streams[0]['duration'])
else:
# Slowest method.

1
modules/videos/test/test_downloader.py
View file

@ -4,7 +4,6 @@ from copy import copy
from unittest import mock
import pytest
from yt_dlp.utils import UnsupportedError
from modules.videos.channel.lib import download_channel
from modules.videos.downloader import VideoDownloader, \

8
modules/videos/test/test_lib.py
View file

@ -1,4 +1,3 @@
import datetime
import pathlib
import shutil
@ -7,10 +6,8 @@ import pytest
from modules.videos import lib
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
from modules.videos.models import Video
from wrolpi.dates import now
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag
from wrolpi.vars import PROJECT_DIR
@ -67,7 +64,7 @@ async def test_video_factory(test_session, video_factory, channel_factory):
assert video.source_id == 'some id'
def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file):
def test_validate_video(test_directory, video_factory, image_bytes_factory, video_file):
"""A video poster will be generated only if the channel permits."""
video_file = video_file.rename(test_directory / 'Channel Name_20050607_1234567890_The Title.mp4')
vid1 = video_factory(with_video_file=video_file)
@ -81,7 +78,8 @@ def test_validate_video(test_session, test_directory, video_factory, image_bytes
assert vid1.poster_path.is_file(), 'Poster path does not exist'
assert vid1.video_path.stem == vid1.poster_path.stem
assert vid1.poster_path.suffix == '.jpg'
assert vid1.upload_date == datetime.datetime(2005, 6, 7, 0, 0, tzinfo=datetime.timezone.utc)
# File name date is assumed to be local timezone.
assert vid1.upload_date and vid1.upload_date.year == 2005
assert vid1.source_id == '1234567890'
assert vid1.file_group.title == 'The Title'

1
modules/videos/video/test/test_api.py
View file

@ -6,7 +6,6 @@ from json import dumps
import pytest
from modules.videos.models import Video
from modules.videos.video.lib import get_video_for_app
from wrolpi.dates import now

6
modules/zim/downloader.py
View file

@ -1,10 +1,8 @@
from http import HTTPStatus
from typing import List
import bs4
from modules.zim import lib
from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task
from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task, get_html_soup
from wrolpi.downloader import Downloader, Download, DownloadResult
__all__ = ['KiwixCatalogDownloader', 'KiwixZimDownloader', 'kiwix_zim_downloader', 'kiwix_catalog_downloader']
@ -20,7 +18,7 @@ async def fetch_hrefs(url: str) -> List[str]:
if status != HTTPStatus.OK:
raise RuntimeError(f'Failed to fetch file catalog')
soup = bs4.BeautifulSoup(content, 'html.parser')
soup = get_html_soup(content)
downloads = list()
for a_ in soup.find_all('a'):
downloads.append(a_['href'])

4
pi-gen/stage2/04-wrolpi/01-packages
View file

@ -1,5 +1,6 @@
apache2
apache2-dev
apache2-doc
apt-transport-https
autoconf
build-essential
@ -17,8 +18,10 @@ fonts-noto-cjk
fonts-noto-hinted
fonts-noto-unhinted
gcc
gcc-doc
gdal-bin
git
git-doc
gnupg-agent
hostapd
iotop
@ -62,6 +65,7 @@ osm2pgsql
osmium-tool
pkg-config
postgis
postgis-doc
postgresql-15
postgresql-15-postgis-3
postgresql-15-postgis-3-scripts

2
pi-gen/stage2/04-wrolpi/04-run-chroot.sh
View file

@ -43,7 +43,7 @@ chmod 644 /var/www/html/*
# Allow wrolpi user to delete map tiles.
usermod -aG _renderd wrolpi
mkdir /var/cache/renderd/tiles
[ ! -d /var/cache/renderd/tiles ] && mkdir /var/cache/renderd/tiles
chown -R _renderd:_renderd /var/cache/renderd/tiles
# Create the media directory for the wrolpi user.

3
requirements.txt
View file

@ -1,6 +1,5 @@
Pillow==10.0.1
Pint==0.21
pypdf==3.12.1
PyYAML==6.0
SQLAlchemy[asyncio]==1.3.22
aiohttp==3.8.5
@ -13,8 +12,10 @@ libzim==3.1.0
mock==5.0.2
psutil==5.9.5
psycopg2-binary==2.9.6
pypdf==3.12.1
pytest-asyncio==0.21.0
pytest==7.3.1
python-dateutil==2.8.2
python-dotenv==1.0.0
python-magic~=0.4.27
pytz==2023.3

106
scripts/singlefile_to_archive.py Executable file
View file

@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Extracts Readability, screenshot, etc. from a Singlefile."""
import argparse
import json
import logging
import pathlib
import subprocess
import sys
import tempfile
from modules.archive import lib as archive_lib
from wrolpi.cmd import READABILITY_BIN
from wrolpi.common import html_screenshot, format_html_string, logger
logger = logger.getChild('singlefile_to_archive')
def do_readability(html: bytes, url: str) -> dict:
"""Extract the readability dict from the provided HTML."""
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
fh.write(html)
cmd = (READABILITY_BIN, fh.name, url)
output = subprocess.check_output(cmd, timeout=60)
readability = json.loads(output)
return readability
def singlefile_to_archive_files(singlefile: pathlib.Path):
"""Extract Readability files and Screenshot file next to the provided Singlefile."""
contents = singlefile.read_bytes()
readability_html_path = singlefile.with_suffix('.readability.html')
readability_json_path = singlefile.with_suffix('.readability.json')
readability_text_path = singlefile.with_suffix('.readability.txt')
# Only extract Readability if any of the files are missing.
readability = dict()
if not readability_html_path.is_file() or not readability_text_path.is_file() or not readability_json_path.is_file():
url = archive_lib.get_url_from_singlefile(contents)
try:
readability = do_readability(contents, url)
except Exception as e:
logger.error(f'Failed to run readability', exc_info=e)
# The real Readability the user wants.
readability_html = readability.pop('content', None)
if readability and not readability_html_path.is_file() and readability_html:
try:
readability_html = format_html_string(readability_html)
readability_html_path.write_text(readability_html)
except Exception as e:
logger.error(f'Failed to extract readability: {singlefile}', exc_info=e)
# The text from within the Readability, used for searching.
readability_text = readability.pop('textContent', None)
if readability and not readability_text_path.is_file() and readability_text:
readability_text_path.write_text(readability_text)
# Write the JSON last now that content/textContent have been removed.
if readability and not readability_json_path.is_file():
try:
with readability_json_path.open('wt') as fh:
fh.write(json.dumps(readability, indent=2))
except Exception as e:
logger.error(f'Failed to extract Readability text: {singlefile}', exc_info=e)
# Only generate a screenshot if the file is missing.
screenshot_path = singlefile.with_suffix('.png')
if not screenshot_path.is_file():
try:
screenshot = html_screenshot(contents)
screenshot_path.write_bytes(screenshot)
except Exception as e:
logger.error(f'Failed to create screenshot: {singlefile}', exc_info=e)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('files', nargs='+')
parser.add_argument('-v', action='count')
args = parser.parse_args()
if args.v == 0:
logger.setLevel(logging.WARNING)
elif args.v == 1:
logger.setLevel(logging.INFO)
elif args.v >= 2:
logger.setLevel(logging.DEBUG)
logger.debug(f'Debug logging')
if not READABILITY_BIN:
print(f'Cannot find "readability-extractor". Install it.')
sys.exit(1)
files = [pathlib.Path(i) for i in args.files]
invalid_files = [i for i in files if not archive_lib.is_singlefile_file(i)]
if invalid_files:
for i in invalid_files:
print(f'File is not a singlefile: {i}')
sys.exit(1)
for file in files:
singlefile_to_archive_files(file)
print(f'Finished {file}')

8
wrolpi/cmd.py
View file

@ -55,25 +55,22 @@ CPUFREQ_SET_BIN = which('cpufreq-set', '/usr/bin/cpufreq-set')
WGET_BIN = which('wget', '/usr/bin/wget', flag=flags.wget_installed)
# Map
BASH_BIN = which('bash', '/bin/bash', warn=True)
BASH_BIN = which('bash', '/bin/bash')
# Archive
SINGLE_FILE_BIN = which('single-file',
'/usr/bin/single-file', # rpi os
'/usr/local/bin/single-file', # debian
warn=True,
flag=flags.singlefile_installed,
)
CHROMIUM = which('chromium-browser', 'chromium',
'/usr/bin/chromium-browser', # rpi os
'/usr/bin/chromium', # debian
warn=True,
flag=flags.chromium_installed,
)
READABILITY_BIN = which('readability-extractor',
'/usr/bin/readability-extractor', # rpi os
'/usr/local/bin/readability-extractor', # debian
warn=True,
flag=flags.readability_installed,
)
@ -82,8 +79,7 @@ YT_DLP_BIN = which(
'yt-dlp',
'/usr/local/bin/yt-dlp', # Location in docker container
'/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location
warn=True,
flag=flags.yt_dlp_installed,
)
FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe', warn=True, flag=flags.ffprobe_installed)
FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe', flag=flags.ffprobe_installed)
FFMPEG_BIN = which('ffmpeg', '/usr/bin/ffmpeg', flag=flags.ffmpeg_installed)

69
wrolpi/common.py
View file

@ -27,6 +27,7 @@ from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Gener
from urllib.parse import urlparse, urlunsplit
import aiohttp
import bs4
import yaml
from aiohttp import ClientResponse
from bs4 import BeautifulSoup
@ -36,7 +37,6 @@ from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from wrolpi import dates
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS
@ -129,8 +129,8 @@ __all__ = [
'get_config',
'get_download_info',
'get_files_and_directories',
'get_html_soup',
'get_media_directory',
'get_model_by_table_name',
'get_relative_to_media_directory',
'get_warn_once',
'insert_parameter',
@ -138,7 +138,6 @@ __all__ = [
'limit_concurrent',
'logger',
'make_media_directory',
'match_paths_to_suffixes',
'minimize_dict',
'native_only',
'partition',
@ -147,7 +146,6 @@ __all__ = [
'register_refresh_cleanup',
'remove_whitespace',
'run_after',
'sanitize_link',
'set_log_level',
'set_test_config',
'set_test_media_directory',
@ -161,6 +159,8 @@ __all__ = [
'wrol_mode_check',
'wrol_mode_enabled',
'zig_zag',
'html_screenshot',
'format_html_string',
]
# Base is used for all SQLAlchemy models.
@ -200,13 +200,6 @@ class ModelHelper:
Session.object_session(self).flush([self])
def get_model_by_table_name(table_name):
"""Find a model by its table name."""
for klass in Base._decl_class_registry.values():
if hasattr(klass, '__tablename__') and klass.__tablename__ == table_name:
return klass
class tsvector(types.TypeDecorator):
impl = types.UnicodeText
@ -219,12 +212,6 @@ def compile_tsvector(element, compiler, **kw):
URL_CHARS = string.ascii_lowercase + string.digits
def sanitize_link(link: str) -> str:
"""Remove any non-url safe characters, all will be lowercase."""
new_link = ''.join(i for i in str(link).lower() if i in URL_CHARS)
return new_link
class ConfigFile:
"""
This class keeps track of the contents of a config file. You can update the config (and it's file) by
@ -1118,37 +1105,21 @@ def register_refresh_cleanup(func: callable):
async def apply_refresh_cleanup():
# TODO convert all functions to async so refresh functions can be used.
for func in REFRESH_CLEANUP:
start = dates.now()
try:
logger_.info(f'Applying refresh cleanup {func.__name__}')
func()
except Exception as e:
logger_.error(f'Refresh cleanup {func.__name__} failed!', exc_info=e)
if PYTEST:
raise
finally:
elapsed = (dates.now() - start).total_seconds()
logger_.warning(f'After refresh cleanup {func.__name__} took {int(elapsed)} seconds')
slow_message = f'After refresh cleanup {func.__name__} took %(elapsed)s seconds'
with slow_logger(5, slow_message, logger__=logger_):
try:
logger_.info(f'Applying refresh cleanup {func.__name__}')
func()
except Exception as e:
logger_.error(f'Refresh cleanup {func.__name__} failed!', exc_info=e)
if PYTEST:
raise
# Sleep to catch cancel.
await asyncio.sleep(0)
@iterify(tuple)
def match_paths_to_suffixes(paths: List[pathlib.Path], suffix_groups: List[Tuple[str]]):
paths = paths.copy()
for group in suffix_groups:
match = None
for suffix in group:
# Compare all suffixes in the group to the path's name. Yield the first path that shares the first
# suffix. If no suffix matches any path, yield None.
match = next(filter(lambda i: i.name.endswith(suffix), paths), None)
if match:
paths.pop(paths.index(match))
break
yield match
def chunks(it: Iterable, size: int):
"""Split an iterable into iterables of the defined length."""
it = iter(it)
@ -1210,7 +1181,7 @@ def timer(name):
@contextlib.contextmanager
def slow_logger(max_seconds: int, message: str, logger__=logger, level=logging.WARNING):
def slow_logger(max_seconds: int, message: str, logger__=logger_, level=logging.WARNING):
"""Only logs when the context duration exceeds the `max_seconds`."""
before = now()
try:
@ -1472,8 +1443,13 @@ def url_strip_host(url: str) -> str:
return url or '/'
def extract_html_text(html: str) -> str:
def get_html_soup(html: Union[bytes, str]) -> bs4.BeautifulSoup:
soup = BeautifulSoup(html, features='html.parser')
return soup
def extract_html_text(html: str) -> str:
soup = get_html_soup(html)
# kill all script and style elements
for script in soup(["script", "style"]):
@ -1538,6 +1514,9 @@ def format_json_file(file: pathlib.Path, indent: int = 2):
def format_html_string(html: str) -> str:
if not html.strip():
raise RuntimeError('Refusing to format empty HTMl string.')
soup = BeautifulSoup(html, features='lxml')
return soup.prettify()

89
wrolpi/dates.py
View file

@ -1,8 +1,10 @@
import logging
import re
from datetime import datetime, timezone, timedelta
from enum import Enum
from typing import Union
import dateutil.parser
import pytz
from sqlalchemy import types
@ -41,18 +43,84 @@ def strftime(dt: datetime) -> str:
return dt.isoformat()
def strptime(dt: str) -> datetime:
if isinstance(dt, str) and dt.endswith(' 00:00'):
# URL decoding removed +
dt = f'{dt[:26]}+00:00'
return datetime.fromisoformat(dt).astimezone(pytz.UTC)
def strpdate(dt: str) -> datetime:
if '-' in dt:
return datetime.strptime(dt, '%Y-%m-%d').replace(tzinfo=pytz.UTC)
"""
Attempt to parse a datetime string. Tries to find a Timezone, if possible. DB requires timezone.
return datetime.strptime(dt, '%Y%m%d').replace(tzinfo=pytz.UTC)
TODO replace this in Python 3.11 https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat
"""
try:
if dt.count('/') == 2:
# No timezone info.
try:
a, b, c = dt.split('/')
if len(c) == 4:
# Assume m/d/Y
return datetime.strptime(dt, '%m/%d/%Y')
if len(a) == 4:
# Y/m/d
return datetime.strptime(dt, '%Y/%m/%d')
except ValueError:
pass
elif dt.count('-') == 2 and len(dt) <= 10:
# No timezone info.
a, b, c = dt.split('-')
if len(a) == 4:
# Y-m-d
return datetime.strptime(dt, '%Y-%m-%d')
if dt.startswith('D:'):
# D:20221226113758-07'00, D:20200205184724, etc.
dt2 = dt.replace("'", '').rstrip('+')
try:
return datetime.strptime(dt2, "D:%Y%m%d%H%M%S%z")
except ValueError:
pass
return datetime.strptime(dt2, "D:%Y%m%d%H%M%S")
if dt.count('-') == 0 and dt.count('/') == 0 and dt.isdigit() and len(dt) == 8:
# Assume %Y%m%d date
return datetime.strptime(dt, '%Y%m%d')
if dt.count('.') == 2 and len(dt) <= 9:
# yyyy.mm.dd or dd.mm.yyyy
try:
a, b, c = dt.split('.')
if len(a) == 4:
return datetime.strptime(dt, '%Y.%m.%d')
if len(c) == 4:
return datetime.strptime(dt, '%d.%m.%Y')
except ValueError:
pass
try:
# Fri Jun 17 2022 19:24:52 (from Singlefile)
return datetime.strptime(dt, '%a %b %d %Y %H:%M:%S')
except ValueError:
pass
if len(dt) == 4 and dt.isdigit():
# Assume this is a year
return datetime.strptime(dt, '%Y').replace(tzinfo=pytz.UTC)
if ',' in dt and (dt.endswith('AM') or dt.endswith('PM')):
try:
# Tuesday, October 19, 1999 3:41:01 PM
return datetime.strptime(dt, '%A, %B %d, %Y %I:%M:%S %p')
except ValueError:
pass
try:
# Tue, October 19, 1999 3:41:01 PM
return datetime.strptime(dt, '%a, %B %d, %Y %I:%M:%S %p')
except ValueError:
pass
# Last, try third-party module to parse ISO 8601 datetime.
return dateutil.parser.isoparse(dt)
except Exception as e:
raise RuntimeError(f'Unable to parse datetime string: {dt}') from e
def strftime_ms(dt: datetime) -> str:
@ -73,6 +141,7 @@ def seconds_to_timestamp(seconds: Union[int, float]) -> str:
>>> seconds_to_timestamp(5)
# '00:00:05'
>>> seconds_to_timestamp(60)
# '00:01:00'
>>> seconds_to_timestamp(86400)
# '1d 00:00:00'

1
wrolpi/files/downloader.py
View file

@ -1,4 +1,3 @@
import asyncio
import pathlib
import traceback
from abc import ABC

5
wrolpi/files/ebooks.py
View file

@ -4,11 +4,10 @@ import logging
import pathlib
from typing import List, Optional, Tuple
import bs4
from sqlalchemy import Column, Integer, String, BigInteger, ForeignKey, or_, and_
from sqlalchemy.orm import relationship
from wrolpi.common import ModelHelper, Base, register_modeler
from wrolpi.common import ModelHelper, Base, register_modeler, get_html_soup
from wrolpi.db import get_db_session
from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
from wrolpi.files.models import FileGroup
@ -66,7 +65,7 @@ class EBookData:
def extract_text(html: str) -> str:
soup = bs4.BeautifulSoup(html, features='html.parser')
soup = get_html_soup(html)
text = soup.get_text()
return text

98
wrolpi/files/lib.py
View file

@ -1,4 +1,5 @@
import asyncio
import dataclasses
import datetime
import functools
import glob
@ -412,14 +413,16 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
AND file_group.size = EXCLUDED.size
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
)
RETURNING id, indexed
RETURNING id, file_group.indexed AS old_indexed, indexed
'''
curs.execute(stmt)
need_index = [i[0] for i in curs.fetchall() if i[1] is False]
if need_index:
refresh_logger.info(f'Invalidated indexes of {len(need_index)} file groups near {str(chunk[0])}')
results = list(curs.fetchall())
# Count the files that used to be indexed, but need to be re-indexed.
invalidated_files = len([i['id'] for i in results if i['old_indexed'] and not i['indexed']])
if invalidated_files:
refresh_logger.info(f'Invalidated indexes of {invalidated_files} file groups near {chunk[0]}')
else:
refresh_logger.debug(f'Upserted {len(chunk)} files near {str(chunk[0])}')
refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}')
if non_primary_files:
# New files may have been added which change what primary paths exist. Delete any file_groups which
@ -456,9 +459,15 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
directories, files = partition(lambda i: i.is_dir(), exists)
while directories:
directory = directories.pop(0)
new_files, new_directories = get_files_and_directories(directory)
try:
new_files, new_directories = get_files_and_directories(directory)
except FileNotFoundError as e:
# Directory may have been deleted during refresh.
logger.warning(f'Cannot refresh directory because it is missing: {directory}', exc_info=e)
continue
directories.extend(new_directories)
files.extend(new_files)
refresh_logger.debug(f'Discovered {len(new_files)} files in {directory}')
if len(files) >= 100:
# Wait until there are enough files to perform the upsert.
try:
@ -475,6 +484,8 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
except Exception as e:
refresh_logger.error(f'Failed to upsert files', exc_info=e)
refresh_logger.info('Finished discovering files. Will now remove deleted files from DB...')
with get_db_curs(commit=True) as curs:
wheres = ''
if paths:
@ -512,6 +523,9 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
if isinstance(paths, pathlib.Path):
paths = [paths, ]
idempotency = now()
REFRESH['idempotency'] = idempotency
refreshing_all_files = False
with flags.refreshing:
@ -523,8 +537,6 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
if send_events:
Events.send_global_refresh_started()
idempotency = now()
# Add all files in the media directory to the DB.
paths = paths or [get_media_directory()]
@ -591,6 +603,7 @@ async def apply_indexers():
file_groups = session.query(FileGroup).filter(FileGroup.indexed != True).limit(20)
file_groups: List[FileGroup] = list(file_groups)
file_group = None
processed = 0
for file_group in file_groups:
processed += 1
@ -606,7 +619,8 @@ async def apply_indexers():
# Sleep to catch cancel.
await asyncio.sleep(0)
refresh_logger.debug(f'Indexed {processed} files')
if file_group:
refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}')
if processed < 20:
# Processed less than the limit, don't do the next query.
@ -969,9 +983,52 @@ async def remove_file_group_tag(file_group_id: int, file_group_primary_path: str
file_group.remove_tag(tag, session)
def get_refresh_progress():
with get_db_curs() as curs:
curs.execute('''
@dataclasses.dataclass
class RefreshProgress:
counted_files: int = 0
counting: bool = False
discovery: bool = False
indexed: int = 0
indexing: bool = False
modeled: int = 0
modeling: bool = False
cleanup: bool = False
refreshing: bool = False
total_file_groups: int = 0
unindexed: int = 0
def __json__(self):
d = dict(
counted_files=self.counted_files,
counting=self.counting,
discovery=self.discovery,
indexed=self.indexed,
indexing=self.indexing,
modeled=self.modeled,
modeling=self.modeling,
cleanup=self.cleanup,
refreshing=self.refreshing,
total_file_groups=self.total_file_groups,
unindexed=self.unindexed,
)
return d
def get_refresh_progress() -> RefreshProgress:
idempotency = REFRESH.get('idempotency')
if idempotency:
stmt = '''
SELECT
-- Sum all the files in each FileGroup.
SUM(json_array_length(files)) FILTER (WHERE idempotency=%(idempotency)s) AS "total_file_groups",
COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency=%(idempotency)s) AS "indexed",
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency=%(idempotency)s) AS "modeled"
FROM file_group
'''
else:
# Idempotency has not yet been declared.
stmt = '''
SELECT
-- Sum all the files in each FileGroup.
SUM(json_array_length(files)) AS "total_file_groups",
@ -979,25 +1036,28 @@ def get_refresh_progress():
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
FROM file_group
''')
'''
with get_db_curs() as curs:
curs.execute(stmt, dict(idempotency=idempotency))
results = dict(curs.fetchone())
# TODO counts are wrong if we are not refreshing all files.
status = dict(
progress = RefreshProgress(
counted_files=REFRESH.get('counted_files', 0),
counting=flags.refresh_counting.is_set(),
discovery=flags.refresh_discovery.is_set(),
indexed=results['indexed'],
indexed=int(results['indexed'] or 0),
indexing=flags.refresh_indexing.is_set(),
modeled=results['modeled'],
modeled=int(results['modeled'] or 0),
modeling=flags.refresh_modeling.is_set(),
cleanup=flags.refresh_cleanup.is_set(),
refreshing=flags.refreshing.is_set(),
total_file_groups=results['total_file_groups'],
unindexed=results['unindexed'],
total_file_groups=int(results['total_file_groups'] or 0),
unindexed=int(results['unindexed'] or 0),
)
return status
return progress
async def estimate_search(search_str: str, tag_names: List[str]):

16
wrolpi/files/models.py
View file

@ -65,17 +65,20 @@ class FileGroup(ModelHelper, Base):
__tablename__ = 'file_group'
id: int = Column(BigInteger, primary_key=True)
# the Path of the file that can be modeled or indexed.
author = Column(String) # name of the author, maybe even a URL
data = Column(FancyJSON) # populated by the modeler
files = Column(FancyJSON, nullable=False) # populated during discovery
idempotency = Column(TZDateTime)
indexed = Column(Boolean, default=lambda: False, nullable=False)
idempotency = Column(TZDateTime) # used to track which files need to be deleted during refresh
indexed = Column(Boolean, default=lambda: False, nullable=False) # wrolpi.files.lib.apply_indexers
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
model = Column(String) # "video", "archive", "ebook", etc.
modification_datetime = Column(TZDateTime)
# the Path of the file that can be modeled or indexed.
modification_datetime = Column(TZDateTime) # the modification date of the file on disk
primary_path: pathlib.Path = Column(MediaPathType, nullable=False, unique=True)
published_datetime = Column(TZDateTime) # the date the creator published this file
published_modified_datetime = Column(TZDateTime) # the date the publisher modified this file
size = Column(BigInteger, default=lambda: 0)
title = Column(String)
title = Column(String) # user-displayable title
tag_files: InstrumentedList = relationship('TagFile', cascade='all')
@ -114,6 +117,9 @@ class FileGroup(ModelHelper, Base):
'tags': tags,
'title': self.title,
'key': self.primary_path,
'author': self.author,
'published_datetime': self.published_datetime,
'published_modified_datetime': self.published_modified_datetime,
}
return d

89
wrolpi/files/pdfs.py
View file

@ -3,6 +3,9 @@ import dataclasses
import pathlib
from typing import Generator
import pytz
from wrolpi import dates
from wrolpi.common import register_modeler, logger, truncate_generator_bytes, truncate_object_bytes, \
split_lines_by_length
from wrolpi.db import get_db_session
@ -24,31 +27,8 @@ __all__ = ['get_pdf_metadata', 'get_words', 'pdf_modeler']
class PDFMetadata:
title: str = None
author: str = None
text: str = None
def create_index(path):
file_title = Indexer.get_title(path)
reader = PdfReader(path)
data = get_pdf_metadata(reader, path)
if path.stat().st_size > FILE_MAX_PDF_SIZE:
logger.warning(f'PDF too large to fully index: {path}')
return data.title, data.author, file_title, None
words = ''
try:
# PDFs are complex, don't fail to create title index if text extraction fails.
words = '\n'.join(truncate_generator_bytes(get_words(reader, path), FILE_MAX_TEXT_SIZE))
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
words = split_lines_by_length(words)
except Exception as e:
logger.error(f'Failed to index {path}', exc_info=e)
if PYTEST:
raise
return data.title, data.author, file_title, words
published_datetime: str = None
modification_datetime: str = None
def get_pdf_metadata(reader: PdfReader, path: pathlib.Path) -> PDFMetadata:
@ -60,8 +40,31 @@ def get_pdf_metadata(reader: PdfReader, path: pathlib.Path) -> PDFMetadata:
data.title = Indexer.get_title(path)
return data
data.title = reader.metadata.title
data.author = reader.metadata.author
if (title := reader.metadata.title) and title.lower() != 'unknown':
data.title = title
if (author := reader.metadata.author) and author.lower() != 'unknown':
data.author = author
if reader.metadata.creation_date_raw:
try:
str(reader.metadata.creation_date)
data.published_datetime = reader.metadata.creation_date
except ValueError:
# pypdf could not parse the date, lets give it a try...
data.published_datetime = dates.strpdate(reader.metadata.creation_date_raw)
if not data.published_datetime.tzinfo:
data.published_datetime = data.published_datetime.astimezone(pytz.UTC)
if reader.metadata.modification_date_raw:
try:
str(reader.metadata.modification_date)
data.modification_datetime = reader.metadata.modification_date
except ValueError:
# pypdf could not parse the date, lets give it a try...
data.modification_datetime = dates.strpdate(reader.metadata.modification_date_raw)
if not data.modification_datetime.tzinfo:
data.modification_datetime = data.modification_datetime.astimezone(pytz.UTC)
return data
@ -117,12 +120,32 @@ async def pdf_modeler():
# Assume the first PDF is the only PDF.
pdf_file = pdf_files[0]['path']
try:
title, author, file_title, contents = create_index(pdf_file)
file_group.data = {'author': author}
file_group.title = file_group.a_text = title
file_group.b_text = author
file_group.c_text = file_title
file_group.d_text = contents
file_title = Indexer.get_title(pdf_file)
reader = PdfReader(pdf_file)
metadata = get_pdf_metadata(reader, pdf_file)
words = ''
if pdf_file.stat().st_size > FILE_MAX_PDF_SIZE:
logger.warning(f'PDF too large to fully index: {pdf_file}')
else:
try:
# PDFs are complex, don't fail to create title index if text extraction fails.
words = '\n'.join(truncate_generator_bytes(get_words(reader, pdf_file), FILE_MAX_TEXT_SIZE))
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
words = split_lines_by_length(words)
except Exception as e:
logger.error(f'Failed to index {pdf_file}', exc_info=e)
if PYTEST:
raise
file_group.title = file_group.a_text = metadata.title or file_title
file_group.author = file_group.b_text = metadata.author
file_group.c_text = file_title # The name of the file may not match the title in the PDF metadata.
file_group.d_text = words or None
file_group.published_datetime = metadata.published_datetime
file_group.published_modified_datetime = metadata.modification_datetime
file_group.model = 'pdf'
except Exception as e:
logger.error(f'Failed to index PDF {pdf_file}', exc_info=e)

2
wrolpi/files/test/test_lib.py
View file

@ -13,7 +13,7 @@ import pytest
from PIL import Image
from wrolpi.common import timer
from wrolpi.dates import now, from_timestamp
from wrolpi.dates import now
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
from wrolpi.files import lib, indexers
from wrolpi.files.models import FileGroup

10
wrolpi/files/test/test_pdfs.py
View file

@ -13,8 +13,8 @@ async def test_pdf_title(test_session, example_pdf):
"""PDF modeler extracts a PDF's title and contents."""
await files_lib.refresh_files()
with mock.patch('wrolpi.files.pdfs.create_index') as mock_create_index:
mock_create_index.side_effect = Exception('Should not index twice')
with mock.patch('wrolpi.files.pdfs.get_pdf_metadata') as mock_get_pdf_metadata:
mock_get_pdf_metadata.side_effect = Exception('Should not index twice')
await files_lib.refresh_files()
pdf: FileGroup = test_session.query(FileGroup).one()
@ -23,7 +23,11 @@ async def test_pdf_title(test_session, example_pdf):
assert pdf.b_text == 'roland'
assert pdf.c_text == 'pdf example pdf'
assert pdf.d_text and pdf.d_text.startswith('Page one\n') and len(pdf.d_text) == 467
assert pdf.data and pdf.data.get('author') == 'roland'
# Metadata within the PDF.
assert pdf.author
assert pdf.published_datetime
# The date the file was modified.
assert pdf.modification_datetime
@pytest.mark.asyncio

6
wrolpi/root_api.py
View file

@ -17,13 +17,13 @@ from sanic_ext.extensions.openapi import openapi
from vininfo import Vin
from vininfo.details._base import VinDetails
from wrolpi import admin, status, flags, schema
from wrolpi import admin, status, flags, schema, dates
from wrolpi import tags
from wrolpi.admin import HotspotStatus
from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, LOG_LEVEL, \
set_global_log_level
from wrolpi.dates import now, strptime
from wrolpi.dates import now
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, APIError, HotspotError, InvalidDownload, \
HotspotPasswordTooShort, NativeOnly, InvalidConfig
@ -411,7 +411,7 @@ async def get_statistics(_):
@validate(query=schema.EventsRequest)
async def feed(_: Request, query: schema.EventsRequest):
start = now()
after = None if query.after == 'None' else strptime(query.after)
after = None if query.after == 'None' else dates.strpdate(query.after)
events = get_events(after)
return json_response(dict(events=events, now=start))

6
wrolpi/scrape_downloader.py
View file

@ -2,9 +2,7 @@ import pathlib
from http import HTTPStatus
from urllib.parse import urlparse
import bs4
from wrolpi.common import aiohttp_session, logger
from wrolpi.common import aiohttp_session, logger, get_html_soup
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.vars import DEFAULT_HTTP_HEADERS
@ -90,7 +88,7 @@ class ScrapeHTMLDownloader(Downloader):
# Find all anchors, search for matching files, and more URLs to search.
try:
soup = bs4.BeautifulSoup(content, 'html.parser')
soup = get_html_soup(content)
except Exception as e:
logger.error(f'Failed to parse HTML from {url}', exc_info=e)
continue

22
wrolpi/test/test_common.py
View file

@ -302,28 +302,6 @@ def test_escape_file_name(name, expected):
raise AssertionError(f'Escape of {repr(name)}: {repr(result)} != {repr(expected)}')
@pytest.mark.parametrize(
'paths,suffix_groups,expected', [
([], [], ()),
(['foo.mp4', 'foo.info.json'], [('.mp4',)], ('foo.mp4',)),
(['foo.mp4', 'foo.info.json'], [('.mp4',), ('.info.json',)], ('foo.mp4', 'foo.info.json')),
(['foo.mp4', 'foo.info.json'], [('.mp4',), ('.nope',), ('.info.json',)], ('foo.mp4', None, 'foo.info.json')),
(['foo.mp4', 'foo.info.json', 'extra.txt'], [('.mp4',)], ('foo.mp4',)),
(['foo.mp4'], [('.mp4', '.flv'), ('.info.json',)], ('foo.mp4', None)),
(
# Two files are matched to the closest suffix.
['foo.info.json', 'bar.json'],
[('.info.json',), ('.json',)], # TODO longest suffix must be first
('foo.info.json', 'bar.json'),
),
]
)
def test_match_paths_to_suffixes(paths, suffix_groups, expected):
paths = [pathlib.Path(i) for i in paths]
expected = tuple(pathlib.Path(i) if i else None for i in expected)
assert (i := common.match_paths_to_suffixes(paths, suffix_groups)) == expected, f'{i} != {expected}'
def test_truncate_object_bytes():
"""
Objects can be truncated (lists will be shortened) so they will fit in tsvector columns.

63
wrolpi/test/test_dates.py
View file

@ -1,7 +1,8 @@
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
import pytest
import pytz
from dateutil.tz import tzoffset
from sqlalchemy import Column, Integer
from wrolpi import dates
@ -86,18 +87,54 @@ def test_timedelta_to_timestamp(td, expected):
@pytest.mark.parametrize('dt,expected', [
('2023-02-26T02:27:26.408944+00:00', datetime(2023, 2, 26, 2, 27, 26, 408944, tzinfo=pytz.UTC)),
('2023-02-26T02:27:26.408944 00:00', datetime(2023, 2, 26, 2, 27, 26, 408944, tzinfo=pytz.UTC)),
])
def test_strptime(dt, expected):
assert dates.strptime(dt) == expected
@pytest.mark.parametrize('dt,expected', [
('20010101', datetime(2001, 1, 1, tzinfo=pytz.UTC)),
('20050607', datetime(2005, 6, 7, tzinfo=pytz.UTC)),
('2006-07-08', datetime(2006, 7, 8, tzinfo=pytz.UTC)),
('2007-8-9', datetime(2007, 8, 9, tzinfo=pytz.UTC)),
('20010101', datetime(2001, 1, 1)),
('20050607', datetime(2005, 6, 7)),
('2006-07-08', datetime(2006, 7, 8)),
('2007-8-9', datetime(2007, 8, 9)),
('2023-10-19T05:53:24+00:00', datetime(2023, 10, 19, 5, 53, 24, tzinfo=pytz.UTC)),
('2022-09-27T00:40:19.000Z', datetime(2022, 9, 27, 0, 40, 19, tzinfo=pytz.UTC)),
('06/17/2022', datetime(2022, 6, 17)),
('14.3.2002', datetime(2002, 3, 14)),
('2002.3.14', datetime(2002, 3, 14)),
('2022/8/1', datetime(2022, 8, 1)),
('2022-8-1', datetime(2022, 8, 1)),
('2023-08-22T02:45:59+0000', datetime(2023, 8, 22, 2, 45, 59, tzinfo=pytz.UTC)),
('2023-08-29T15:29-0400', datetime(2023, 8, 29, 15, 29, tzinfo=tzoffset(None, -14400))),
('2023-08-29T15:29+0400', datetime(2023, 8, 29, 15, 29, tzinfo=tzoffset(None, 14400))),
('2011-11-04', datetime(2011, 11, 4, 0, 0)),
('20111104', datetime(2011, 11, 4, 0, 0)),
('2011-11-04T00:05:23', datetime(2011, 11, 4, 0, 5, 23)),
('2011-11-04T00:05:23Z', datetime(2011, 11, 4, 0, 5, 23, tzinfo=pytz.UTC)),
('20111104T000523', datetime(2011, 11, 4, 0, 5, 23)),
('2011-W01-2T00:05:23.283', datetime(2011, 1, 4, 0, 5, 23, 283000)),
('2011-11-04 00:05:23.283', datetime(2011, 11, 4, 0, 5, 23, 283000)),
('2011-11-04 00:05:23.283+00:00', datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=pytz.UTC)),
('2011-11-04T00:05:23+04:00', datetime(2011, 11, 3, 20, 5, 23, tzinfo=pytz.UTC)),
('2000', datetime(2000, 1, 1, tzinfo=pytz.UTC)),
('Tuesday, October 19, 1999 3:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)),
('Tue, October 19, 1999 3:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)),
('Tue, October 19, 1999 03:41:01 PM', datetime(1999, 10, 19, 15, 41, 1)),
# From SingleFile.
('Fri Jun 17 2022 19:24:52', datetime(2022, 6, 17, 19, 24, 52)),
# PDFs are the Wild West...
("D:20221226113758-07'00", datetime(2022, 12, 26, 11, 37, 58, tzinfo=timezone(timedelta(days=-1, seconds=61200)))),
('D:20200205184724', datetime(2020, 2, 5, 18, 47, 24)),
("D:20091019120104+", datetime(2009, 10, 19, 12, 1, 4)),
])
def test_strpdate(dt, expected):
assert dates.strpdate(dt) == expected
def test_invalid_strpdate():
with pytest.raises(RuntimeError):
dates.strpdate('1')
with pytest.raises(RuntimeError):
dates.strpdate('asdf')
with pytest.raises(RuntimeError):
dates.strpdate('13/1/2008')
# Not a real date.
with pytest.raises(RuntimeError):
dates.strpdate('2001-2-31')
with pytest.raises(RuntimeError):
dates.strpdate('2001-2-30')

2
wrolpi/vars.py
View file

@ -46,8 +46,6 @@ def truthy_arg(value: str) -> bool:
# Get the media directory from the environment.
DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')
MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY))
if not MEDIA_DIRECTORY.is_dir() and not PYTEST:
print(f'Media directory does not exist! {MEDIA_DIRECTORY}', file=sys.stderr)
CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config'
MODULES_DIR: Path = PROJECT_DIR / 'modules'