mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
Converted ArchiveDownloader to use Downloader.process_runner so archiving can be canceled.
Moving Archive download functions to ArchiveDownloader. Wrapping all do_download with cancel_wrapper which allows canceling of downloads.
This commit is contained in:
parent
a0a4e139fa
commit
7611cf8ea1
13 changed files with 350 additions and 266 deletions
|
|
@ -1,22 +1,26 @@
|
|||
import asyncio
|
||||
import json
|
||||
import pathlib
|
||||
import tempfile
|
||||
from abc import ABC
|
||||
from typing import List
|
||||
from typing import List, Tuple
|
||||
|
||||
from selenium import webdriver
|
||||
from sqlalchemy import not_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM, READABILITY_BIN
|
||||
from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
|
||||
slow_logger
|
||||
slow_logger, html_screenshot
|
||||
from wrolpi.db import optional_session, get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST
|
||||
from wrolpi.vars import PYTEST, DOCKERIZED
|
||||
from . import lib
|
||||
from .api import bp # noqa
|
||||
from .errors import InvalidArchive
|
||||
from .lib import is_singlefile_file, get_title_from_html
|
||||
from .lib import is_singlefile_file, get_title_from_html, request_archive, SINGLEFILE_HEADER
|
||||
from .models import Archive, Domain
|
||||
|
||||
PRETTY_NAME = 'Archive'
|
||||
|
|
@ -37,7 +41,14 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
if download.attempts > 3:
|
||||
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
|
||||
|
||||
archive: Archive = await lib.do_archive(download.url)
|
||||
if DOCKERIZED or PYTEST:
|
||||
# Perform the archive in the Archive docker container. (Typically in the development environment).
|
||||
singlefile, readability, screenshot = await request_archive(download.url)
|
||||
else:
|
||||
# Perform the archive using locally installed executables.
|
||||
singlefile, readability, screenshot = await self.do_archive(download)
|
||||
|
||||
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
|
||||
|
||||
if download.settings and (tag_names := download.settings.get('tag_names')):
|
||||
for name in tag_names:
|
||||
|
|
@ -53,6 +64,99 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
archives = list(session.query(Archive).filter(Archive.url.in_(urls)))
|
||||
return archives
|
||||
|
||||
async def do_singlefile(self, download: Download) -> bytes:
|
||||
"""Create a Singlefile from the archive's URL."""
|
||||
cmd = (str(SINGLE_FILE_BIN),
|
||||
download.url,
|
||||
'--browser-executable-path', CHROMIUM,
|
||||
'--browser-args', '["--no-sandbox"]',
|
||||
'--dump-content')
|
||||
return_code, _, stdout = await self.process_runner(
|
||||
download.url,
|
||||
cmd,
|
||||
pathlib.Path('/home/wrolpi'),
|
||||
)
|
||||
if return_code != 0:
|
||||
raise RuntimeError(f'Archive singlefile exited with {return_code}')
|
||||
|
||||
return stdout
|
||||
|
||||
async def do_readability(self, download: Download, html: bytes) -> dict:
|
||||
"""Extract the readability dict from the provided HTML."""
|
||||
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
|
||||
fh.write(html)
|
||||
|
||||
cmd = (READABILITY_BIN, fh.name, download.url)
|
||||
logger.debug(f'readability cmd: {cmd}')
|
||||
return_code, logs, stdout = await self.process_runner(
|
||||
download.url,
|
||||
cmd,
|
||||
pathlib.Path('/home/wrolpi'),
|
||||
)
|
||||
if return_code == 0:
|
||||
readability = json.loads(stdout)
|
||||
logger.debug(f'done readability for {download.url}')
|
||||
return readability
|
||||
else:
|
||||
logger.error(f'Failed to extract readability for {download.url}')
|
||||
raise RuntimeError(f'Failed to extract readability for {download.url}')
|
||||
|
||||
@staticmethod
|
||||
async def do_screenshot_url(download: Download) -> bytes:
|
||||
"""Use Chromium to get a screenshot of the Download's URL."""
|
||||
# Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of
|
||||
# the page.
|
||||
options = webdriver.ChromeOptions()
|
||||
options.add_argument('headless')
|
||||
options.add_argument('disable-gpu')
|
||||
options.add_argument('window-size=1280x720')
|
||||
|
||||
driver = webdriver.Chrome(chrome_options=options)
|
||||
driver.get(download.url)
|
||||
screenshot = driver.get_screenshot_as_png()
|
||||
return screenshot
|
||||
|
||||
async def do_archive(self, download: Download) -> Tuple[bytes, dict, bytes]:
|
||||
"""Use locally installed executables to create an Archive.
|
||||
|
||||
Creates a Singlefile, readability file, and screenshot file.
|
||||
|
||||
@warning: Will not raise errors if readability or screenshot cannot be extracted.
|
||||
"""
|
||||
singlefile = await self.do_singlefile(download)
|
||||
|
||||
if SINGLEFILE_HEADER.encode() not in singlefile[:1000]:
|
||||
raise RuntimeError(f'Singlefile created was invalid: {download.ur}')
|
||||
|
||||
# Extract Readability from the Singlefile.
|
||||
try:
|
||||
readability = await self.do_readability(download, singlefile)
|
||||
except RuntimeError:
|
||||
# Readability is not required.
|
||||
readability = None
|
||||
|
||||
screenshot = b''
|
||||
try:
|
||||
# Screenshot the Singlefile first.
|
||||
screenshot = html_screenshot(singlefile)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to screenshot file for {download.url}', exc_info=e)
|
||||
|
||||
if not screenshot:
|
||||
# Screenshot of singlefile failed. Download the URL again.
|
||||
try:
|
||||
screenshot = await self.do_screenshot_url(download)
|
||||
except Exception as e:
|
||||
# Screenshot failed.
|
||||
logger.error(f'Failed to screenshot {download.url}', exc_info=e)
|
||||
|
||||
singlefile_len = len(singlefile) if singlefile else None
|
||||
readability_len = len(readability) if readability else None
|
||||
screenshot_len = len(screenshot) if screenshot else None
|
||||
logger.debug(f'do_archive of {download.url} finished: {singlefile_len=} {readability_len=} {screenshot_len=}')
|
||||
|
||||
return singlefile, readability, screenshot
|
||||
|
||||
|
||||
archive_downloader = ArchiveDownloader()
|
||||
|
||||
|
|
|
|||
|
|
@ -3,26 +3,21 @@ import gzip
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple, List
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from sqlalchemy import asc
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.archive.models import Domain, Archive
|
||||
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
|
||||
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post, \
|
||||
from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \
|
||||
format_html_string, split_lines_by_length
|
||||
from wrolpi.dates import now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy
|
||||
from wrolpi.tags import tag_names_to_file_group_sub_select
|
||||
from wrolpi.vars import DOCKERIZED, PYTEST
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -146,84 +141,11 @@ async def request_archive(url: str) -> Tuple[str, Optional[str], Optional[str]]:
|
|||
return singlefile, readability, screenshot
|
||||
|
||||
|
||||
def local_singlefile(url: str):
|
||||
"""Run the single-file executable to create an HTML file archive."""
|
||||
if not SINGLE_FILE_BIN or not SINGLE_FILE_BIN.is_file():
|
||||
raise FileNotFoundError(f'single-file not found. Is it installed?')
|
||||
|
||||
cmd = (str(SINGLE_FILE_BIN),
|
||||
url,
|
||||
'--browser-executable-path', CHROMIUM,
|
||||
'--browser-args', '["--no-sandbox"]',
|
||||
'--dump-content')
|
||||
logger.debug(f'archive cmd: {cmd}')
|
||||
output = subprocess.check_output(cmd, timeout=60 * 3)
|
||||
logger.debug(f'done archiving for {url}')
|
||||
return output
|
||||
|
||||
|
||||
def local_screenshot(url: str) -> bytes:
|
||||
"""Take a screenshot of the URL using chromedriver."""
|
||||
logger.info(f'Screenshot: {url}')
|
||||
|
||||
# Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page.
|
||||
options = webdriver.ChromeOptions()
|
||||
options.add_argument('headless')
|
||||
options.add_argument('disable-gpu')
|
||||
options.add_argument('window-size=1280x720')
|
||||
|
||||
driver = webdriver.Chrome(chrome_options=options)
|
||||
try:
|
||||
driver.get(url)
|
||||
png = driver.get_screenshot_as_png()
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to screenshot {url}', exc_info=e)
|
||||
return b''
|
||||
return png
|
||||
|
||||
|
||||
def local_extract_readability(path: str, url: str) -> dict:
|
||||
"""Extract the readability from an HTML file, typically from single-file."""
|
||||
logger.info(f'readability for {url}')
|
||||
if not READABILITY_BIN.is_file():
|
||||
raise FileNotFoundError(f'Readability extractor not found')
|
||||
|
||||
cmd = (READABILITY_BIN, path, url)
|
||||
logger.debug(f'readability cmd: {cmd}')
|
||||
output = subprocess.check_output(cmd, timeout=60 * 3)
|
||||
output = json.loads(output)
|
||||
logger.debug(f'done readability for {url}')
|
||||
return output
|
||||
|
||||
|
||||
def local_archive(url: str):
|
||||
"""Perform an archive of the provided URL using local resources (without the Archive docker container)."""
|
||||
# Archives must be performed in the wrolpi home directory because chrome saves the screenshot there, and will
|
||||
# raise an error if global $HOME is not a real user directory. :(
|
||||
with chdir('/home/wrolpi', with_home=True):
|
||||
singlefile = local_singlefile(url)
|
||||
with tempfile.NamedTemporaryFile('wb') as fh:
|
||||
fh.write(singlefile)
|
||||
readability = local_extract_readability(fh.name, url)
|
||||
screenshot = local_screenshot(url)
|
||||
singlefile = singlefile.decode()
|
||||
return singlefile, readability, screenshot
|
||||
|
||||
|
||||
async def do_archive(url: str) -> Archive:
|
||||
async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes) -> Archive:
|
||||
"""
|
||||
Perform the real archive request to the archiving service. Store the resulting data into files. Create an Archive
|
||||
record in the DB. Create Domain if missing.
|
||||
Convert results from ArchiveDownloader into real files. Create Archive record.
|
||||
"""
|
||||
logger.info(f'Archiving {url}')
|
||||
|
||||
if DOCKERIZED or PYTEST:
|
||||
# Perform the archive in the Archive docker container. (Typically in the development environment).
|
||||
singlefile, readability, screenshot = await request_archive(url)
|
||||
else:
|
||||
# Perform the archive using locally installed executables.
|
||||
singlefile, readability, screenshot = local_archive(url)
|
||||
|
||||
readability = readability.copy() if readability else readability
|
||||
# First try to get the title from Readability.
|
||||
title = readability.get('title') if readability else None
|
||||
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@ import tempfile
|
|||
from datetime import datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import mock
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from modules.archive import lib
|
||||
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, do_archive, get_domains
|
||||
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, model_archive_result, \
|
||||
get_domains
|
||||
from modules.archive.models import Archive, Domain
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files import lib as files_lib
|
||||
|
|
@ -18,51 +18,48 @@ from wrolpi.root_api import CustomJSONEncoder
|
|||
from wrolpi.test.common import skip_circleci
|
||||
|
||||
|
||||
def make_fake_request_archive(readability=True, screenshot=True, title=True):
|
||||
async def fake_request_archive(_):
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
|
||||
Image.new('RGB', (1, 1), color='grey').save(fh)
|
||||
fh.seek(0)
|
||||
image = fh.read()
|
||||
singlefile = '<html><!--\n Page saved with SingleFile-->\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
r = dict(
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='test readability textContent',
|
||||
title='ジにてこちら' if title else None,
|
||||
) if readability else None
|
||||
s = image if screenshot else None
|
||||
return singlefile, r, s
|
||||
|
||||
return fake_request_archive
|
||||
def make_fake_archive_result(readability=True, screenshot=True, title=True):
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
|
||||
Image.new('RGB', (1, 1), color='grey').save(fh)
|
||||
fh.seek(0)
|
||||
image = fh.read()
|
||||
singlefile = '<html><!--\n Page saved with SingleFile-->\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
r = dict(
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='test readability textContent',
|
||||
title='ジにてこちら' if title else None,
|
||||
) if readability else None
|
||||
s = image if screenshot else None
|
||||
return singlefile, r, s
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_screenshot(test_session):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)):
|
||||
archive = await do_archive('https://example.com')
|
||||
assert isinstance(archive.singlefile_path, pathlib.Path)
|
||||
assert isinstance(archive.readability_path, pathlib.Path)
|
||||
# Screenshot was empty
|
||||
assert archive.screenshot_path is None
|
||||
singlefile, readability, screenshot = make_fake_archive_result(screenshot=False)
|
||||
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
assert isinstance(archive.singlefile_path, pathlib.Path)
|
||||
assert isinstance(archive.readability_path, pathlib.Path)
|
||||
# Screenshot was empty
|
||||
assert archive.screenshot_path is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_readability(test_session):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(readability=False)):
|
||||
archive = await do_archive('https://example.com')
|
||||
assert isinstance(archive.singlefile_path, pathlib.Path)
|
||||
assert isinstance(archive.screenshot_path, pathlib.Path)
|
||||
# Readability empty
|
||||
assert archive.readability_path is None
|
||||
assert archive.readability_txt_path is None
|
||||
singlefile, readability, screenshot = make_fake_archive_result(readability=False)
|
||||
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
assert isinstance(archive.singlefile_path, pathlib.Path)
|
||||
assert isinstance(archive.screenshot_path, pathlib.Path)
|
||||
# Readability empty
|
||||
assert archive.readability_path is None
|
||||
assert archive.readability_txt_path is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict(test_session):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
d = (await do_archive('https://example.com')).dict()
|
||||
assert isinstance(d, dict)
|
||||
json.dumps(d, cls=CustomJSONEncoder)
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
d = (await model_archive_result('https://example.com', singlefile, readability, screenshot)).dict()
|
||||
assert isinstance(d, dict)
|
||||
json.dumps(d, cls=CustomJSONEncoder)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -70,11 +67,9 @@ async def test_relationships(test_session, test_directory, example_singlefile):
|
|||
with get_db_session(commit=True) as session:
|
||||
url = 'https://wrolpi.org:443'
|
||||
domain = get_or_create_domain(session, url)
|
||||
archive = Archive(
|
||||
file_group=FileGroup.from_paths(test_session, example_singlefile),
|
||||
url=url,
|
||||
domain_id=domain.id,
|
||||
)
|
||||
archive = Archive.from_paths(test_session, example_singlefile)
|
||||
archive.url = url
|
||||
archive.domain_id = domain.id
|
||||
session.add(archive)
|
||||
session.flush()
|
||||
|
||||
|
|
@ -284,22 +279,22 @@ def test_get_domains(test_session, archive_factory):
|
|||
|
||||
@pytest.mark.asycio
|
||||
async def test_new_archive(test_session, fake_now):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
fake_now(datetime(2000, 1, 1))
|
||||
archive1 = await do_archive('https://example.com')
|
||||
# Everything is filled out.
|
||||
assert isinstance(archive1, Archive)
|
||||
assert archive1.archive_datetime is not None
|
||||
assert isinstance(archive1.singlefile_path, pathlib.Path)
|
||||
assert isinstance(archive1.readability_path, pathlib.Path)
|
||||
assert isinstance(archive1.readability_txt_path, pathlib.Path)
|
||||
assert isinstance(archive1.screenshot_path, pathlib.Path)
|
||||
assert archive1.file_group.title == 'ジにてこちら'
|
||||
assert archive1.url is not None
|
||||
assert archive1.domain is not None
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
fake_now(datetime(2000, 1, 1))
|
||||
archive1 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
# Everything is filled out.
|
||||
assert isinstance(archive1, Archive)
|
||||
assert archive1.archive_datetime is not None
|
||||
assert isinstance(archive1.singlefile_path, pathlib.Path)
|
||||
assert isinstance(archive1.readability_path, pathlib.Path)
|
||||
assert isinstance(archive1.readability_txt_path, pathlib.Path)
|
||||
assert isinstance(archive1.screenshot_path, pathlib.Path)
|
||||
assert archive1.file_group.title == 'ジにてこちら'
|
||||
assert archive1.url is not None
|
||||
assert archive1.domain is not None
|
||||
|
||||
# The actual files were dumped and read correctly. The HTML/JSON files are reformatted.
|
||||
assert archive1.singlefile_path.read_text() == '''<html>
|
||||
# The actual files were dumped and read correctly. The HTML/JSON files are reformatted.
|
||||
assert archive1.singlefile_path.read_text() == '''<html>
|
||||
<!--
|
||||
Page saved with SingleFile-->
|
||||
<body>
|
||||
|
|
@ -313,7 +308,7 @@ async def test_new_archive(test_session, fake_now):
|
|||
</body>
|
||||
</html>
|
||||
'''
|
||||
assert archive1.readability_path.read_text() == '''<html>
|
||||
assert archive1.readability_path.read_text() == '''<html>
|
||||
<body>
|
||||
<p>
|
||||
test readability content
|
||||
|
|
@ -321,53 +316,47 @@ async def test_new_archive(test_session, fake_now):
|
|||
</body>
|
||||
</html>
|
||||
'''
|
||||
with open(archive1.readability_txt_path) as fh:
|
||||
assert fh.read() == 'test readability textContent'
|
||||
assert archive1.readability_json_path.read_text() == '''{
|
||||
with open(archive1.readability_txt_path) as fh:
|
||||
assert fh.read() == 'test readability textContent'
|
||||
assert archive1.readability_json_path.read_text() == '''{
|
||||
"title": "\\u30b8\\u306b\\u3066\\u3053\\u3061\\u3089",
|
||||
"url": "https://example.com"
|
||||
}'''
|
||||
|
||||
fake_now(datetime(2000, 1, 2))
|
||||
archive2 = await do_archive('https://example.com')
|
||||
# Domain is reused.
|
||||
assert archive1.domain == archive2.domain
|
||||
fake_now(datetime(2000, 1, 2))
|
||||
archive2 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
# Domain is reused.
|
||||
assert archive1.domain == archive2.domain
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_title_from_html(test_session, fake_now):
|
||||
fake_now(datetime(2000, 1, 1))
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
archive = await do_archive('https://example.com')
|
||||
assert archive.file_group.title == 'ジにてこちら'
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
assert archive.file_group.title == 'ジにてこちら'
|
||||
|
||||
async def fake_request_archive(_):
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
r = dict(
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
)
|
||||
s = b'screenshot data'
|
||||
return singlefile, r, s
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
readability = dict(
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
)
|
||||
screenshot = b'screenshot data'
|
||||
|
||||
fake_now(datetime(2000, 1, 2))
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive = await do_archive('https://example.com')
|
||||
assert archive.file_group.title == 'some title'
|
||||
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
assert archive.file_group.title == 'some title'
|
||||
|
||||
async def fake_request_archive(_):
|
||||
singlefile = '<html></html>'
|
||||
r = dict(
|
||||
content=f'<html>missing a title</html>',
|
||||
textContent='',
|
||||
)
|
||||
s = b'screenshot data'
|
||||
return singlefile, r, s
|
||||
singlefile = '<html></html>'
|
||||
readability = dict(
|
||||
content=f'<html>missing a title</html>',
|
||||
textContent='',
|
||||
)
|
||||
screenshot = b'screenshot data'
|
||||
|
||||
fake_now(datetime(2000, 1, 3))
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive = await do_archive('https://example.com')
|
||||
assert archive.file_group.title is None
|
||||
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
assert archive.file_group.title is None
|
||||
|
||||
|
||||
@skip_circleci
|
||||
|
|
@ -411,8 +400,8 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b
|
|||
"""
|
||||
fake_now(datetime(2000, 1, 1))
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
archive1 = await do_archive('https://example.com')
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
archive1 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
|
||||
assert archive1.file_group.title == 'ジにてこちら'
|
||||
|
||||
|
|
@ -427,18 +416,15 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b
|
|||
assert str(archive1.screenshot_path.relative_to(test_directory)) == \
|
||||
'archive/example.com/2000-01-01-00-00-00_ジにてこちら.png'
|
||||
|
||||
async def fake_request_archive(_):
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n</html>' # no title in HTML
|
||||
r = dict(
|
||||
# No Title from Readability.
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='test readability textContent',
|
||||
)
|
||||
s = image_bytes_factory()
|
||||
return singlefile, r, s
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n</html>' # no title in HTML
|
||||
readability = dict(
|
||||
# No Title from Readability.
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='test readability textContent',
|
||||
)
|
||||
screenshot = image_bytes_factory()
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive2 = await do_archive('https://example.com')
|
||||
archive2 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
|
||||
assert str(archive2.singlefile_path.relative_to(test_directory)) == \
|
||||
'archive/example.com/2000-01-01-00-00-00_NA.html'
|
||||
|
|
@ -451,18 +437,15 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b
|
|||
assert str(archive2.screenshot_path.relative_to(test_directory)) == \
|
||||
'archive/example.com/2000-01-01-00-00-00_NA.png'
|
||||
|
||||
async def fake_request_archive(_):
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>dangerous ;\\//_title</html></html>'
|
||||
r = dict(
|
||||
# No Title from Readability.
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
)
|
||||
s = image_bytes_factory()
|
||||
return singlefile, r, s
|
||||
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>dangerous ;\\//_title</html></html>'
|
||||
readability = dict(
|
||||
# No Title from Readability.
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='<html>test readability textContent</html>',
|
||||
)
|
||||
screenshot = image_bytes_factory()
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive3 = await do_archive('https://example.com')
|
||||
archive3 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
|
||||
assert str(archive3.singlefile_path.relative_to(test_directory)) == \
|
||||
'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.html'
|
||||
|
|
@ -620,16 +603,14 @@ def test_archive_order(test_session, test_directory, archive_factory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_download_index(test_session, test_directory, image_file):
|
||||
async def test_archive_download_index(test_session, test_directory, image_bytes_factory):
|
||||
"""An Archive is indexed when it is downloaded."""
|
||||
with mock.patch('modules.archive.lib.request_archive') as mock_request_archive:
|
||||
singlefile = '<html><title>the singlefile</title></html>'
|
||||
readability = dict(
|
||||
content='<html>the readability</html>',
|
||||
textContent='the readability',
|
||||
)
|
||||
mock_request_archive.return_value = (singlefile, readability, image_file.read_bytes())
|
||||
archive = await lib.do_archive('https://example.com')
|
||||
singlefile = '<html><title>the singlefile</title></html>'
|
||||
readability = dict(
|
||||
content='<html>the readability</html>',
|
||||
textContent='the readability',
|
||||
)
|
||||
archive = await lib.model_archive_result('https://example.com', singlefile, readability, image_bytes_factory())
|
||||
|
||||
assert isinstance(archive, lib.Archive), 'Did not get an archive'
|
||||
assert archive.singlefile_path and archive.singlefile_path.exists() and \
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ def mock_video_prepare_filename():
|
|||
@pytest.fixture
|
||||
def mock_video_process_runner():
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_process_runner.return_value = (0, {'stdout': None, 'stderr': None})
|
||||
mock_process_runner.return_value = (0, {'stdout': None, 'stderr': None}, None)
|
||||
yield mock_process_runner
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -230,8 +230,12 @@ class VideoDownloader(Downloader, ABC):
|
|||
destination = settings.get('destination')
|
||||
if not channel and destination:
|
||||
# Destination may override the real Channel.
|
||||
channel = get_channel(directory=destination, return_dict=False)
|
||||
found_channel = 'download_settings_directory'
|
||||
try:
|
||||
channel = get_channel(directory=destination, return_dict=False)
|
||||
found_channel = 'download_settings_directory'
|
||||
except UnknownChannel:
|
||||
# Destination must not be a channel.
|
||||
pass
|
||||
|
||||
local_channel_id = settings.get('channel_id')
|
||||
channel_url = settings.get('channel_url')
|
||||
|
|
@ -298,7 +302,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
'--ppa', 'Merger+ffmpeg_o1:-strict -2',
|
||||
url,
|
||||
)
|
||||
return_code, logs = await self.process_runner(url, cmd, out_dir)
|
||||
return_code, logs, _ = await self.process_runner(url, cmd, out_dir)
|
||||
|
||||
stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
|
||||
stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue