mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Chrome attempts to screenshot in the current directory, but if the current directory is not a real user directory, the screenshot fail.s
This commit is contained in:
parent
1d9fe2f649
commit
6f58a2b563
3 changed files with 84 additions and 9 deletions
|
|
@ -14,7 +14,7 @@ from selenium import webdriver
|
|||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.archive.models import Domain, Archive
|
||||
from wrolpi.common import get_media_directory, logger, chunks, extract_domain
|
||||
from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir
|
||||
from wrolpi.dates import now, strptime_ms, strftime_ms
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
|
||||
from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
|
||||
|
|
@ -144,6 +144,7 @@ def local_screenshot(url: str) -> bytes:
|
|||
# Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page.
|
||||
options = webdriver.ChromeOptions()
|
||||
options.add_argument('headless')
|
||||
options.add_argument('disable-gpu')
|
||||
options.add_argument('window-size=1280x720')
|
||||
|
||||
driver = webdriver.Chrome(chrome_options=options)
|
||||
|
|
@ -177,13 +178,16 @@ def local_archive(url: str):
|
|||
"""
|
||||
Perform an archive of the provided URL using local resources (without the Archive docker container).
|
||||
"""
|
||||
singlefile = local_singlefile(url)
|
||||
with tempfile.NamedTemporaryFile('wb') as fh:
|
||||
fh.write(singlefile)
|
||||
readability = local_extract_readability(fh.name, url)
|
||||
screenshot = local_screenshot(url)
|
||||
singlefile = singlefile.decode()
|
||||
return singlefile, readability, screenshot
|
||||
# Archives must be performed in the wrolpi home directory because chrome saves the screenshot there, and will
|
||||
# raise an error if global $HOME is not a real user directory. :(
|
||||
with chdir('/home/wrolpi', with_home=True):
|
||||
singlefile = local_singlefile(url)
|
||||
with tempfile.NamedTemporaryFile('wb') as fh:
|
||||
fh.write(singlefile)
|
||||
readability = local_extract_readability(fh.name, url)
|
||||
screenshot = local_screenshot(url)
|
||||
singlefile = singlefile.decode()
|
||||
return singlefile, readability, screenshot
|
||||
|
||||
|
||||
def do_archive(url: str) -> Archive:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import contextlib
|
|||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import queue
|
||||
import re
|
||||
|
|
@ -526,3 +527,42 @@ def partition(pred, iterable):
|
|||
"""
|
||||
t1, t2 = tee(iterable)
|
||||
return list(filter(pred, t2)), list(filterfalse(pred, t1))
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def chdir(directory: Union[pathlib.Path, str, None] = None, with_home: bool = False):
|
||||
"""
|
||||
Change the Python working directory in this context. If no directory is passed, a TemporaryDirectory will be used.
|
||||
|
||||
This also changes the $HOME environment variable to match.
|
||||
"""
|
||||
home_exists = 'HOME' in os.environ
|
||||
home = os.environ.get('HOME')
|
||||
cwd = os.getcwd()
|
||||
if directory is None:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
try:
|
||||
os.chdir(d)
|
||||
if with_home:
|
||||
os.environ['HOME'] = d
|
||||
yield
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
if home_exists:
|
||||
os.environ['HOME'] = home
|
||||
else:
|
||||
del os.environ['HOME']
|
||||
return
|
||||
|
||||
try:
|
||||
os.chdir(directory)
|
||||
if with_home:
|
||||
os.environ['HOME'] = str(directory)
|
||||
yield
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
if home_exists:
|
||||
os.environ['HOME'] = home
|
||||
else:
|
||||
del os.environ['HOME']
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import date, datetime
|
||||
|
||||
import pytest
|
||||
from sanic_openapi import doc
|
||||
|
||||
from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter
|
||||
from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir
|
||||
from wrolpi.dates import set_timezone, now
|
||||
from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone
|
||||
from wrolpi.schema import validate_data
|
||||
|
|
@ -315,3 +318,31 @@ class TestCommon(unittest.TestCase):
|
|||
def test_api_param_limiter(i, expected):
|
||||
limiter = api_param_limiter(100) # should never return an integer greater than 100.
|
||||
assert limiter(i) == expected
|
||||
|
||||
|
||||
def test_chdir():
|
||||
"""
|
||||
The current working directory can be changed temporarily using the `chdir` context manager.
|
||||
"""
|
||||
original = os.getcwd()
|
||||
home = os.environ.get('HOME')
|
||||
assert home
|
||||
|
||||
with chdir():
|
||||
assert os.getcwd() != original
|
||||
assert str(os.getcwd()).startswith('/tmp')
|
||||
assert os.environ['HOME'] != os.getcwd()
|
||||
# Replace $HOME
|
||||
with chdir(with_home=True):
|
||||
assert os.getcwd() != original
|
||||
assert str(os.getcwd()).startswith('/tmp')
|
||||
assert os.environ['HOME'] == os.getcwd()
|
||||
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
# Without replacing $HOME
|
||||
with chdir(pathlib.Path(d), with_home=True):
|
||||
assert os.getcwd() == d
|
||||
assert os.environ['HOME'] == os.getcwd()
|
||||
with chdir(pathlib.Path(d)):
|
||||
assert os.getcwd() == d
|
||||
assert os.environ['HOME'] != os.getcwd()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue