From 30affa20cd00c8ea7d801d53ae08efcce9d9d705 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 6 Jan 2022 14:20:21 -0700 Subject: [PATCH] Using archive request when testing. --- modules/archive/lib.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 875ebe41..fc2eecff 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -18,7 +18,7 @@ from wrolpi.common import get_media_directory, logger, chunks, extract_domain from wrolpi.dates import now, strptime_ms, strftime_ms from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive -from wrolpi.vars import DOCKERIZED +from wrolpi.vars import DOCKERIZED, PYTEST logger = logger.getChild(__name__) @@ -114,6 +114,9 @@ def request_archive(url: str): def local_singlefile(url: str): + """ + Run the single-file executable to create an HTML file archive. + """ single_file_path = pathlib.Path('/usr/bin/single-file') if not single_file_path.is_file(): raise FileNotFoundError(f'single-file not found at {single_file_path}') @@ -130,6 +133,9 @@ def local_singlefile(url: str): def local_screenshot(url: str) -> bytes: + """ + Take a screenshot of the URL using selenium. + """ logger.info(f'Screenshot: {url}') # Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page. @@ -148,6 +154,9 @@ def local_screenshot(url: str) -> bytes: def local_extract_readability(path: str, url: str) -> dict: + """ + Extract the readability from an HTML file, typically from single-file. + """ logger.info(f'readability for {url}') readability_path = pathlib.Path('/usr/bin/readability-extractor') if not readability_path.is_file(): @@ -183,7 +192,7 @@ def do_archive(url: str) -> Archive: singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \ get_new_archive_files(url) - if DOCKERIZED: + if DOCKERIZED or PYTEST: # Perform the archive in the Archive docker container. (Typically in the development environment). singlefile, readability, screenshot = request_archive(url) else: