mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Handling when a Singlefile is found that was not created by WROLPi.
This commit is contained in:
parent
1edc2ba4e6
commit
c5538222ef
4 changed files with 107 additions and 36 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import datetime
|
||||
import json
|
||||
import pathlib
|
||||
|
|
@ -84,3 +85,28 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
return archive
|
||||
|
||||
return _
|
||||
|
||||
|
||||
SINGLEFILE_CONTENTS = '''<html><!--
|
||||
Page saved with SingleFile
|
||||
url: https://example.com
|
||||
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
|
||||
--><head><meta charset="utf-8">
|
||||
|
||||
<meta name="msapplication-TileColor" content="#2d89ef">
|
||||
<meta name="theme-color" content="#000000">
|
||||
|
||||
<title>the title</title>
|
||||
|
||||
<body>
|
||||
some body contents
|
||||
</body>
|
||||
|
||||
</html>
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def singlefile_contents() -> str:
|
||||
"""Return a short HTML string that contains an example Singlefile file."""
|
||||
return copy.deepcopy(SINGLEFILE_CONTENTS)
|
||||
|
|
|
|||
|
|
@ -488,13 +488,29 @@ def is_archive_file(path: pathlib.Path) -> bool:
|
|||
return path.is_file() and path.suffix.lower() in ARCHIVE_SUFFIXES and bool(ARCHIVE_MATCHER.match(path.name))
|
||||
|
||||
|
||||
SINGLEFILE_HEADER = '''<!--
|
||||
Page saved with SingleFile'''
|
||||
|
||||
|
||||
def is_singlefile_file(path: pathlib.Path) -> bool:
|
||||
"""
|
||||
Archive singlefile files are expected to start with the following: %Y-%m-%d-%H-%M-%S
|
||||
they must end with .html, but never with .readability.html.
|
||||
"""
|
||||
return path.is_file() and path.suffix.lower() == '.html' and not path.name.endswith('.readability.html') \
|
||||
and bool(ARCHIVE_MATCHER.match(path.name))
|
||||
if not path.is_file() or \
|
||||
path.suffix.lower() != '.html' or \
|
||||
path.name.lower().endswith('.readability.html') or \
|
||||
path.stat().st_size == 0:
|
||||
return False
|
||||
if ARCHIVE_MATCHER.match(path.name):
|
||||
# A file name matching exactly the Archive format from WROLPi should always be an Archive.
|
||||
return True
|
||||
with path.open('rt') as fh:
|
||||
# Lastly, try to read the header of this HTML file.
|
||||
header = fh.read(500)
|
||||
if SINGLEFILE_HEADER in header:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_domains():
|
||||
|
|
|
|||
|
|
@ -18,6 +18,11 @@ from wrolpi.media_path import MediaPathType
|
|||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['Archive', 'Domain']
|
||||
|
||||
MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
|
||||
MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
|
||||
|
||||
|
||||
class Archive(Base, ModelHelper):
|
||||
__tablename__ = 'archive'
|
||||
|
|
@ -144,39 +149,37 @@ class Archive(Base, ModelHelper):
|
|||
"""
|
||||
Read the start of the singlefile (if any) and decode the Archive information.
|
||||
"""
|
||||
match_url = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
|
||||
match_date = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
|
||||
if not self.singlefile_path or not self.singlefile_file:
|
||||
return
|
||||
path = self.singlefile_file.path if self.singlefile_file else self.singlefile_path
|
||||
|
||||
if self.singlefile_path:
|
||||
with self.singlefile_path.open('rt') as fh:
|
||||
head = fh.read(500)
|
||||
if 'Page saved with SingleFile' not in head:
|
||||
return
|
||||
try:
|
||||
if match := match_url.findall(head):
|
||||
self.url = match[0].strip()
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get URL from singlefile {self.singlefile_path}', exc_info=e)
|
||||
try:
|
||||
if match := match_date.findall(head):
|
||||
dt = match[0].strip()
|
||||
dt = ' '.join(dt.split(' ')[:5])
|
||||
# SingleFile uses GMT.
|
||||
dt = datetime.datetime.strptime(
|
||||
dt,
|
||||
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
|
||||
).replace(tzinfo=pytz.timezone('GMT'))
|
||||
self.archive_datetime = dt
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get archive date from singlefile {self.singlefile_path}', exc_info=e)
|
||||
with path.open('rt') as fh:
|
||||
head = fh.read(500)
|
||||
if 'Page saved with SingleFile' not in head:
|
||||
return
|
||||
try:
|
||||
if match := MATCH_URL.findall(head):
|
||||
self.url = match[0].strip()
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
|
||||
try:
|
||||
if match := MATCH_DATE.findall(head):
|
||||
dt = match[0].strip()
|
||||
dt = ' '.join(dt.split(' ')[:5])
|
||||
# SingleFile uses GMT.
|
||||
dt = datetime.datetime.strptime(
|
||||
dt,
|
||||
'%a %b %d %Y %H:%M:%S' # Fri Jun 17 2022 19:24:52
|
||||
).replace(tzinfo=pytz.timezone('GMT'))
|
||||
self.archive_datetime = dt
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
|
||||
|
||||
def read_readability_data(self):
|
||||
"""Read the Readability JSON file, apply its contents to this record."""
|
||||
from modules.archive.lib import get_or_create_domain, get_title_from_html
|
||||
|
||||
readability_json_file = self.readability_json_file
|
||||
if not readability_json_file or not readability_json_file.path.is_file():
|
||||
logger.warning(f'Archive must have a readability json file: {self.singlefile_file.path}')
|
||||
logger.warning(f'{self.singlefile_file.path} does not have an info json file')
|
||||
return
|
||||
|
||||
try:
|
||||
|
|
@ -187,21 +190,30 @@ class Archive(Base, ModelHelper):
|
|||
except Exception as e:
|
||||
raise InvalidArchive() from e
|
||||
|
||||
domain = None
|
||||
if url:
|
||||
domain = get_or_create_domain(Session.object_session(self), url)
|
||||
|
||||
if not title:
|
||||
title = get_title_from_html(self.singlefile_path.read_text())
|
||||
|
||||
self.url = url
|
||||
self.title = title
|
||||
|
||||
def apply_domain(self):
|
||||
"""Get the domain from the URL."""
|
||||
from modules.archive.lib import get_or_create_domain
|
||||
domain = None
|
||||
if self.url:
|
||||
domain = get_or_create_domain(Session.object_session(self), self.url)
|
||||
# Clear domain if the URL is missing.
|
||||
self.domain_id = domain.id if domain else None
|
||||
|
||||
def apply_title(self):
|
||||
"""Get the title from the Singlefile, if its missing."""
|
||||
from modules.archive.lib import get_title_from_html
|
||||
if self.singlefile_path and not self.title:
|
||||
self.title = get_title_from_html(self.singlefile_path.read_text())
|
||||
|
||||
def validate(self):
|
||||
try:
|
||||
self.read_readability_data()
|
||||
self.read_singlefile_data()
|
||||
self.apply_domain()
|
||||
self.apply_title()
|
||||
self.validated = True
|
||||
except Exception as e:
|
||||
logger.warning(f'Unable to validate {self}', exc_info=e)
|
||||
|
|
|
|||
|
|
@ -565,6 +565,23 @@ def test_is_archive_file(name, expected, test_directory):
|
|||
assert lib.is_archive_file(path) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'name,expected', [
|
||||
('foo', False),
|
||||
('2000-01-01-00-00-00_Some NA.html', True),
|
||||
('2000-01-01-00-00-00_Some NA.readability.html', False),
|
||||
('2000-01-01-00-00-00_ジにてこちら.html', True),
|
||||
('2000-01-01-00-00-00_Some Title.readability.txt', False),
|
||||
('foo.txt', False),
|
||||
('foo.html', True),
|
||||
]
|
||||
)
|
||||
def test_is_singlefile_file(name, expected, make_files_structure, singlefile_contents):
|
||||
"""A Singlefile may not be created by WROLPi, an HTML file created by Singlefile can also be an Archive."""
|
||||
path, = make_files_structure({name: singlefile_contents})
|
||||
assert lib.is_singlefile_file(path) == expected
|
||||
|
||||
|
||||
@skip_circleci
|
||||
def test_migrate_archive_files(test_session, archive_directory, archive_factory):
|
||||
"""Test that migration of all old Archive formatted files are migrated."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue