Handling when a Singlefile is found that was not created by WROLPi.

This commit is contained in:
Roland 2022-11-02 14:37:57 -06:00
parent 1edc2ba4e6
commit c5538222ef
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
4 changed files with 107 additions and 36 deletions

View file

@ -1,3 +1,4 @@
import copy
import datetime
import json
import pathlib
@ -84,3 +85,28 @@ def archive_factory(test_session, archive_directory, make_files_structure):
return archive
return _
SINGLEFILE_CONTENTS = '''<html><!--
Page saved with SingleFile
url: https://example.com
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">
<meta name="msapplication-TileColor" content="#2d89ef">
<meta name="theme-color" content="#000000">
<title>the title</title>
<body>
some body contents
</body>
</html>
'''
@pytest.fixture
def singlefile_contents() -> str:
"""Return a short HTML string that contains an example Singlefile file."""
return copy.deepcopy(SINGLEFILE_CONTENTS)

View file

@ -488,13 +488,29 @@ def is_archive_file(path: pathlib.Path) -> bool:
return path.is_file() and path.suffix.lower() in ARCHIVE_SUFFIXES and bool(ARCHIVE_MATCHER.match(path.name))
SINGLEFILE_HEADER = '''<!--
Page saved with SingleFile'''
def is_singlefile_file(path: pathlib.Path) -> bool:
"""
Archive singlefile files are expected to start with the following: %Y-%m-%d-%H-%M-%S
they must end with .html, but never with .readability.html.
"""
return path.is_file() and path.suffix.lower() == '.html' and not path.name.endswith('.readability.html') \
and bool(ARCHIVE_MATCHER.match(path.name))
if not path.is_file() or \
path.suffix.lower() != '.html' or \
path.name.lower().endswith('.readability.html') or \
path.stat().st_size == 0:
return False
if ARCHIVE_MATCHER.match(path.name):
# A file name matching exactly the Archive format from WROLPi should always be an Archive.
return True
with path.open('rt') as fh:
# Lastly, try to read the header of this HTML file.
header = fh.read(500)
if SINGLEFILE_HEADER in header:
return True
return False
def get_domains():

View file

@ -18,6 +18,11 @@ from wrolpi.media_path import MediaPathType
logger = logger.getChild(__name__)
__all__ = ['Archive', 'Domain']
MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
class Archive(Base, ModelHelper):
__tablename__ = 'archive'
@ -144,39 +149,37 @@ class Archive(Base, ModelHelper):
"""
Read the start of the singlefile (if any) and decode the Archive information.
"""
match_url = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
match_date = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
if not self.singlefile_path or not self.singlefile_file:
return
path = self.singlefile_file.path if self.singlefile_file else self.singlefile_path
if self.singlefile_path:
with self.singlefile_path.open('rt') as fh:
head = fh.read(500)
if 'Page saved with SingleFile' not in head:
return
try:
if match := match_url.findall(head):
self.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {self.singlefile_path}', exc_info=e)
try:
if match := match_date.findall(head):
dt = match[0].strip()
dt = ' '.join(dt.split(' ')[:5])
# SingleFile uses GMT.
dt = datetime.datetime.strptime(
dt,
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
).replace(tzinfo=pytz.timezone('GMT'))
self.archive_datetime = dt
except Exception as e:
logger.error(f'Could not get archive date from singlefile {self.singlefile_path}', exc_info=e)
with path.open('rt') as fh:
head = fh.read(500)
if 'Page saved with SingleFile' not in head:
return
try:
if match := MATCH_URL.findall(head):
self.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
try:
if match := MATCH_DATE.findall(head):
dt = match[0].strip()
dt = ' '.join(dt.split(' ')[:5])
# SingleFile uses GMT.
dt = datetime.datetime.strptime(
dt,
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
).replace(tzinfo=pytz.timezone('GMT'))
self.archive_datetime = dt
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
def read_readability_data(self):
"""Read the Readability JSON file, apply its contents to this record."""
from modules.archive.lib import get_or_create_domain, get_title_from_html
readability_json_file = self.readability_json_file
if not readability_json_file or not readability_json_file.path.is_file():
logger.warning(f'Archive must have a readability json file: {self.singlefile_file.path}')
logger.warning(f'{self.singlefile_file.path} does not have an info json file')
return
try:
@ -187,21 +190,30 @@ class Archive(Base, ModelHelper):
except Exception as e:
raise InvalidArchive() from e
domain = None
if url:
domain = get_or_create_domain(Session.object_session(self), url)
if not title:
title = get_title_from_html(self.singlefile_path.read_text())
self.url = url
self.title = title
def apply_domain(self):
"""Get the domain from the URL."""
from modules.archive.lib import get_or_create_domain
domain = None
if self.url:
domain = get_or_create_domain(Session.object_session(self), self.url)
# Clear domain if the URL is missing.
self.domain_id = domain.id if domain else None
def apply_title(self):
"""Get the title from the Singlefile, if its missing."""
from modules.archive.lib import get_title_from_html
if self.singlefile_path and not self.title:
self.title = get_title_from_html(self.singlefile_path.read_text())
def validate(self):
try:
self.read_readability_data()
self.read_singlefile_data()
self.apply_domain()
self.apply_title()
self.validated = True
except Exception as e:
logger.warning(f'Unable to validate {self}', exc_info=e)

View file

@ -565,6 +565,23 @@ def test_is_archive_file(name, expected, test_directory):
assert lib.is_archive_file(path) == expected
@pytest.mark.parametrize(
'name,expected', [
('foo', False),
('2000-01-01-00-00-00_Some NA.html', True),
('2000-01-01-00-00-00_Some NA.readability.html', False),
('2000-01-01-00-00-00_ジにてこちら.html', True),
('2000-01-01-00-00-00_Some Title.readability.txt', False),
('foo.txt', False),
('foo.html', True),
]
)
def test_is_singlefile_file(name, expected, make_files_structure, singlefile_contents):
"""A Singlefile may not be created by WROLPi, an HTML file created by Singlefile can also be an Archive."""
path, = make_files_structure({name: singlefile_contents})
assert lib.is_singlefile_file(path) == expected
@skip_circleci
def test_migrate_archive_files(test_session, archive_directory, archive_factory):
"""Test that migration of all old Archive formatted files are migrated."""