mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 02:48:38 -04:00
Including title in archive file names.
This commit is contained in:
Roland
parent
cad3617239
commit
284efedb80
2 changed files with 156 additions and 47 deletions
|
|
@ -6,7 +6,7 @@ import re
|
|||
import subprocess
|
||||
import tempfile
|
||||
from itertools import groupby
|
||||
from typing import Iterator
|
||||
from typing import Iterator, Optional
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
|
@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from modules.archive.models import Domain, Archive
|
||||
from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir
|
||||
from wrolpi.dates import now, strptime_ms, strftime_ms, Seconds
|
||||
from wrolpi.dates import now, strptime_ms, Seconds, strftime
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
|
||||
from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
|
||||
from wrolpi.vars import DOCKERIZED, PYTEST
|
||||
|
|
@ -47,18 +47,20 @@ def get_domain_directory(url: str) -> pathlib.Path:
|
|||
return directory
|
||||
|
||||
|
||||
def get_new_archive_files(url: str):
|
||||
def get_new_archive_files(url: str, title: Optional[str]):
|
||||
"""
|
||||
Create a list of archive files using a shared name schema. Raise an error if any of them exist.
|
||||
"""
|
||||
directory = get_domain_directory(url)
|
||||
dt = strftime_ms(now())
|
||||
dt = now().strftime('%Y-%m-%d-%H-%M-%S')
|
||||
|
||||
singlefile_path = directory / f'{dt}.html'
|
||||
readability_path = directory / f'{dt}-readability.html'
|
||||
readability_txt_path = directory / f'{dt}-readability.txt'
|
||||
readability_json_path = directory / f'{dt}-readability.json'
|
||||
screenshot_path = directory / f'{dt}.png'
|
||||
title = escape_file_name(title or 'NA')
|
||||
prefix = f'{dt}_{title}'
|
||||
singlefile_path = directory / f'{prefix}.html'
|
||||
readability_path = directory / f'{prefix}.readability.html'
|
||||
readability_txt_path = directory / f'{prefix}.readability.txt'
|
||||
readability_json_path = directory / f'{prefix}.readability.json'
|
||||
screenshot_path = directory / f'{prefix}.png'
|
||||
|
||||
ret = (singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path)
|
||||
|
||||
|
|
@ -196,8 +198,6 @@ def do_archive(url: str) -> Archive:
|
|||
record in the DB. Create Domain/URL if missing.
|
||||
"""
|
||||
logger.info(f'Archiving {url}')
|
||||
singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
|
||||
get_new_archive_files(url)
|
||||
|
||||
if DOCKERIZED or PYTEST:
|
||||
# Perform the archive in the Archive docker container. (Typically in the development environment).
|
||||
|
|
@ -206,6 +206,30 @@ def do_archive(url: str) -> Archive:
|
|||
# Perform the archive using locally installed executables.
|
||||
singlefile, readability, screenshot = local_archive(url)
|
||||
|
||||
# First try to get the title from Readability.
|
||||
title = readability.get('title') if readability else None
|
||||
|
||||
if not title and singlefile:
|
||||
# Try to get the title ourselves from the HTML.
|
||||
title = get_title_from_html(singlefile, url=url)
|
||||
if readability:
|
||||
# Readability could not find title, lets use ours.
|
||||
readability['title'] = title
|
||||
|
||||
singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
|
||||
get_new_archive_files(url, title)
|
||||
|
||||
if readability:
|
||||
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
|
||||
with readability_path.open('wt') as fh:
|
||||
fh.write(readability.pop('content'))
|
||||
with readability_txt_path.open('wt') as fh:
|
||||
readability_txt = readability.pop('textContent')
|
||||
fh.write(readability_txt)
|
||||
else:
|
||||
# No readability was returned, so there are no files.
|
||||
readability_txt = readability_path = readability_txt_path = None
|
||||
|
||||
# Store the single-file HTML in its own file.
|
||||
with singlefile_path.open('wt') as fh:
|
||||
fh.write(singlefile)
|
||||
|
|
@ -215,27 +239,8 @@ def do_archive(url: str) -> Archive:
|
|||
else:
|
||||
screenshot_path = None
|
||||
|
||||
# Store the Readability into separate files. This allows the user to view text-only or html articles.
|
||||
title = None
|
||||
readability_txt = None
|
||||
if readability:
|
||||
title = readability.get('title')
|
||||
|
||||
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
|
||||
with readability_path.open('wt') as fh:
|
||||
fh.write(readability.pop('content'))
|
||||
with readability_txt_path.open('wt') as fh:
|
||||
readability_txt = readability.pop('textContent')
|
||||
fh.write(readability_txt)
|
||||
else:
|
||||
readability_path = readability_txt_path = None
|
||||
|
||||
# Always write a JSON file that contains at least the URL.
|
||||
readability = readability or {}
|
||||
# Use the Readability title, or try and extract one from singlefile.
|
||||
if not title and singlefile:
|
||||
title = get_title_from_html(singlefile, url=url)
|
||||
readability['title'] = title
|
||||
readability['url'] = url
|
||||
with readability_json_path.open('wt') as fh:
|
||||
fh.write(json.dumps(readability))
|
||||
|
|
@ -496,3 +501,12 @@ def search(search_str: str, domain: str, limit: int, offset: int):
|
|||
results = get_ranked_models(ranked_ids, Archive)
|
||||
|
||||
return results, total
|
||||
|
||||
|
||||
# These characters are invalid in Windows or Linux.
|
||||
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]')
|
||||
|
||||
|
||||
def escape_file_name(name: str) -> str:
|
||||
"""Replace any invalid characters in a file name with "_"."""
|
||||
return INVALID_FILE_CHARS.sub('', name)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import pathlib
|
|||
from datetime import datetime
|
||||
|
||||
import mock
|
||||
import pytest
|
||||
|
||||
from modules.archive import lib
|
||||
from modules.archive.lib import get_or_create_domain, _refresh_archives, \
|
||||
get_new_archive_files, delete_archive, do_archive, get_domains
|
||||
from modules.archive.models import Archive, Domain
|
||||
|
|
@ -34,23 +36,6 @@ class TestArchive(TestAPI):
|
|||
super().setUp()
|
||||
(pathlib.Path(self.tmp_dir.name) / 'archive').mkdir(exist_ok=True)
|
||||
|
||||
@mock.patch('modules.archive.lib.now', lambda: datetime(2001, 1, 1))
|
||||
def test_get_new_archive_files(self):
|
||||
s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two'))
|
||||
assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html')
|
||||
assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html')
|
||||
assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt')
|
||||
assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json')
|
||||
assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png')
|
||||
|
||||
s, r, t, j, c = get_new_archive_files('https://www.example.com/one')
|
||||
# Leading www. is removed.
|
||||
assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html')
|
||||
assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html')
|
||||
assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt')
|
||||
assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json')
|
||||
assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png')
|
||||
|
||||
@wrap_test_db
|
||||
def test_no_screenshot(self):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)):
|
||||
|
|
@ -328,3 +313,113 @@ def test_get_title_from_html(test_session, fake_now):
|
|||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive = do_archive('example.com')
|
||||
assert archive.title is None
|
||||
|
||||
|
||||
def test_get_new_archive_files(fake_now):
|
||||
"""Archive files have a specific format so they are sorted by datetime, and are near each other."""
|
||||
fake_now(datetime(2001, 1, 1))
|
||||
s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two', None))
|
||||
assert str(s).endswith('archive/example.com/2001-01-01-00-00-00_NA.html')
|
||||
assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.html')
|
||||
assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.txt')
|
||||
assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.json')
|
||||
assert str(c).endswith('archive/example.com/2001-01-01-00-00-00_NA.png')
|
||||
|
||||
s, r, t, j, c = get_new_archive_files('https://www.example.com/one', 'Title')
|
||||
# Leading www. is removed.
|
||||
assert str(s).endswith('archive/example.com/2001-01-01-00-00-00_Title.html')
|
||||
assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.html')
|
||||
assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.txt')
|
||||
assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.json')
|
||||
assert str(c).endswith('archive/example.com/2001-01-01-00-00-00_Title.png')
|
||||
|
||||
|
||||
def test_title_in_filename(test_session, fake_now, test_directory):
|
||||
"""
|
||||
The Archive files have the title in the path.
|
||||
"""
|
||||
fake_now(datetime(2000, 1, 1))
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
archive1 = do_archive('example.com')
|
||||
|
||||
assert archive1.title == 'ジにてこちら'
|
||||
|
||||
assert str(archive1.singlefile_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_ジにてこちら.html'
|
||||
assert str(archive1.readability_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_ジにてこちら.readability.html'
|
||||
assert str(archive1.readability_txt_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_ジにてこちら.readability.txt'
|
||||
assert str(archive1.readability_json_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_ジにてこちら.readability.json'
|
||||
assert str(archive1.screenshot_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_ジにてこちら.png'
|
||||
|
||||
def fake_request_archive(_):
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n</html>' # no title in HTML
|
||||
r = dict(
|
||||
# No Title from Readability.
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
)
|
||||
s = b'screenshot data'
|
||||
return singlefile, r, s
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive2 = do_archive('example.com')
|
||||
|
||||
assert str(archive2.singlefile_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_NA.html'
|
||||
assert str(archive2.readability_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_NA.readability.html'
|
||||
assert str(archive2.readability_txt_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_NA.readability.txt'
|
||||
assert str(archive2.readability_json_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_NA.readability.json'
|
||||
assert str(archive2.screenshot_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_NA.png'
|
||||
|
||||
def fake_request_archive(_):
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>dangerous ;\\//_title</html></html>'
|
||||
r = dict(
|
||||
# No Title from Readability.
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
)
|
||||
s = b'screenshot data'
|
||||
return singlefile, r, s
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive3 = do_archive('example.com')
|
||||
|
||||
assert str(archive3.singlefile_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_dangerous ;_title.html'
|
||||
assert str(archive3.readability_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_dangerous ;_title.readability.html'
|
||||
assert str(archive3.readability_txt_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_dangerous ;_title.readability.txt'
|
||||
assert str(archive3.readability_json_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_dangerous ;_title.readability.json'
|
||||
assert str(archive3.screenshot_path.path.relative_to(test_directory)) == \
|
||||
'archive/2000-01-01-00-00-00_dangerous ;_title.png'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'name,expected', [
|
||||
('', ''),
|
||||
('foo', 'foo'),
|
||||
('foo\\', 'foo'),
|
||||
('foo/', 'foo'),
|
||||
('foo<', 'foo'),
|
||||
('foo>', 'foo'),
|
||||
('foo:', 'foo'),
|
||||
('foo|', 'foo'),
|
||||
('foo"', 'foo'),
|
||||
('foo?', 'foo'),
|
||||
('foo*', 'foo'),
|
||||
('foo&', 'foo&'),
|
||||
]
|
||||
)
|
||||
def test_escape_file_name(name, expected):
|
||||
assert lib.escape_file_name(name) == expected
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue