diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index c01c0fe9..8633e2d6 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -4,7 +4,9 @@ WORKDIR /opt/wrolpi # Update dependencies for the services install. RUN apt update -RUN apt install -y ffmpeg wkhtmltopdf +# ffmpeg for Videos and Latex for OTP PDF. +RUN apt-get install -y ffmpeg texlive-latex-base texlive-latex-extra +RUN pdflatex --version # Install WROLPi COPY main.py /opt/wrolpi/ diff --git a/install.sh b/install.sh index 5d923053..a45e1b5f 100755 --- a/install.sh +++ b/install.sh @@ -47,7 +47,11 @@ curl -fsSL https://deb.nodesource.com/setup_14.x | bash - # Install dependencies apt install -y git apt-transport-https ca-certificates curl gnupg-agent gcc libpq-dev software-properties-common \ postgresql-12 nginx-full nginx-doc python3.8-minimal python3.8-dev python3.8-doc python3.8-venv \ - ffmpeg wkhtmltopdf hostapd nodejs yarn + ffmpeg hostapd nodejs yarn texlive-latex-base texlive-latex-extra chromium-browser chromium-chromedriver + +# Install Archiving tools. +npm install -g "gildas-lormeau/SingleFile#master" +npm install -g 'git+https://github.com/pirate/readability-extractor' # Get the latest WROLPi code git --version diff --git a/main.py b/main.py index 089d8b55..2b909abe 100755 --- a/main.py +++ b/main.py @@ -121,6 +121,7 @@ def periodic_downloads(app: Sanic, loop): if wrol_mode_enabled(): logger.warning(f'Not starting download worker because WROL Mode is enabled.') return + return logger.info('Starting download manager.') # Set all downloads to new. diff --git a/modules/archive/lib.py b/modules/archive/lib.py index b41d9ef7..875ebe41 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -3,11 +3,14 @@ import gzip import json import pathlib import re +import subprocess +import tempfile from itertools import groupby from typing import Iterator import requests from bs4 import BeautifulSoup +from selenium import webdriver from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive @@ -15,6 +18,7 @@ from wrolpi.common import get_media_directory, logger, chunks, extract_domain from wrolpi.dates import now, strptime_ms, strftime_ms from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive +from wrolpi.vars import DOCKERIZED logger = logger.getChild(__name__) @@ -69,6 +73,8 @@ def request_archive(url: str): """ Send a request to the archive service to archive the URL. """ + logger.info(f'Sending archive request to archive service: {url}') + data = {'url': url} try: resp = requests.post(f'{ARCHIVE_SERVICE}/json', json=data, timeout=ARCHIVE_TIMEOUT) @@ -107,15 +113,82 @@ def request_archive(url: str): return singlefile, readability, screenshot +def local_singlefile(url: str): + single_file_path = pathlib.Path('/usr/bin/single-file') + if not single_file_path.is_file(): + raise FileNotFoundError(f'single-file not found at {single_file_path}') + + cmd = (str(single_file_path), + url, + '--browser-executable-path', '/usr/bin/chromium-browser', + '--browser-args', '["--no-sandbox"]', + '--dump-content') + logger.debug(f'archive cmd: {cmd}') + output = subprocess.check_output(cmd, timeout=60 * 3) + logger.debug(f'done archiving for {url}') + return output + + +def local_screenshot(url: str) -> bytes: + logger.info(f'Screenshot: {url}') + + # Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page. + options = webdriver.ChromeOptions() + options.add_argument('headless') + options.add_argument('window-size=1280x720') + + driver = webdriver.Chrome(chrome_options=options) + try: + driver.get(url) + png = driver.get_screenshot_as_png() + except Exception as e: + logger.warning(f'Failed to screenshot {url}', exc_info=e) + return b'' + return png + + +def local_extract_readability(path: str, url: str) -> dict: + logger.info(f'readability for {url}') + readability_path = pathlib.Path('/usr/bin/readability-extractor') + if not readability_path.is_file(): + raise FileNotFoundError(f'Readability extractor not found at {readability_path}') + + cmd = (readability_path, path, url) + logger.debug(f'readability cmd: {cmd}') + output = subprocess.check_output(cmd, timeout=60 * 3) + output = json.loads(output) + logger.debug(f'done readability for {url}') + return output + + +def local_archive(url: str): + """ + Perform an archive of the provided URL using local resources (without the Archive docker container). + """ + singlefile = local_singlefile(url) + with tempfile.NamedTemporaryFile('wb') as fh: + fh.write(singlefile) + readability = local_extract_readability(fh.name, url) + screenshot = local_screenshot(url) + singlefile = singlefile.decode() + return singlefile, readability, screenshot + + def do_archive(url: str) -> Archive: """ Perform the real archive request to the archiving service. Store the resulting data into files. Create an Archive record in the DB. Create Domain/URL if missing. """ + logger.info(f'Archiving {url}') singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \ get_new_archive_files(url) - singlefile, readability, screenshot = request_archive(url) + if DOCKERIZED: + # Perform the archive in the Archive docker container. (Typically in the development environment). + singlefile, readability, screenshot = request_archive(url) + else: + # Perform the archive using locally installed executables. + singlefile, readability, screenshot = local_archive(url) # Store the single-file HTML in its own file. with singlefile_path.open('wt') as fh: diff --git a/modules/otp/api.py b/modules/otp/api.py index e8e79b8d..01a212ce 100644 --- a/modules/otp/api.py +++ b/modules/otp/api.py @@ -7,7 +7,7 @@ from sanic.request import Request from wrolpi.root_api import get_blueprint from wrolpi.schema import validate_doc -from .common import generate_html, generate_pdf, encrypt_otp, decrypt_otp +from .common import generate_pdf, encrypt_otp, decrypt_otp from .schema import EncryptOTPRequest, DecryptOTPRequest bp = get_blueprint('OTP', '/api/otp') @@ -33,11 +33,6 @@ async def post_decrypt_otp(_: Request, data: dict): return response.json(data) -@bp.get('/html') -async def get_new_otp(_: Request): - return response.html(generate_html()) - - @bp.get('/pdf') async def get_new_otp_pdf(_: Request): random_name = ''.join(random.choice(ascii_letters) for _ in range(8)) diff --git a/modules/otp/common.py b/modules/otp/common.py index f5d4eb47..fc1ce393 100644 --- a/modules/otp/common.py +++ b/modules/otp/common.py @@ -1,13 +1,15 @@ import secrets +import subprocess from functools import partial from typing import Dict -import pdfkit - -from wrolpi.common import chunks, remove_whitespace +from wrolpi.common import chunks, remove_whitespace, logger, temporary_directory_path from wrolpi.errors import InvalidOTP, InvalidPlaintext, InvalidCiphertext # These are the only characters we support. + +logger = logger.getChild(__name__) + OTP_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' # The number of characters in each group, separated by spaces. GROUP_SIZE = 5 @@ -36,48 +38,57 @@ def generate_message(): Create a OTP message. Format it for ease of use. """ # Use 20 messages per line because this page will be printed. - return format_message(''.join(generate_char() for _ in range(400)), 20) + return format_message(''.join(generate_char() for _ in range(400)), 16) -PAGE_HTML = ''' - -One Time Pad - Unique, just for you - - -{messages} +ONE_TIME_PAD_TEX = r''' +\documentclass{article} +\usepackage{hyperref} +\usepackage[margin=0.3in]{geometry} +\usepackage{mathptmx} +\title{One Time Pad} -Print this page and distribute the copies (along with the One Time Pad Cheat Sheet -https://lrnsr.co/aY6m) to all members of your group that you trust to receive your encrypted messages. Every -person must have their OWN copy of this "One Time Pad" to encrypt and decrypt messages. -
-Use each message ONLY ONCE. Cut off and burn each message from this paper as it is used. -
-If you want more One Time Pads, simply go here to refresh the page: -https://learningselfreliance.com/one_time_pad. The server will generate a unique page just for you. This page -is not stored on the server, and cannot be retrieved once you close this window! -
-To learn how to use this page, please visit: https://lrnsr.co/H7Za - - +\begin{document} + +\begin{verbatim} +%s +\end{verbatim} + +Print this page and distribute the copies (along with the \href{https://lrnsr.co/aY6m}{One Time Pad Cheat Sheet +https://lrnsr.co/aY6m}) to all members of your group that you trust to receive your encrypted messages. Every person +must have their OWN copy of this “One Time Pad” to encrypt and decrypt messages. +\\ + +\textbf{Use each message ONLY ONCE}. Cut off and burn each message from this paper as it is used. +\\ + +To learn how to use this page, please visit: \href{https://lrnsr.co/H7Za}{https://lrnsr.co/H7Za} + +\end{document} ''' -def generate_html() -> str: +def generate_pdf() -> bytes: """ - Create an HTML One-Time Pad page. This page will have instructions on how to use the OTP. + Create a PDF One Time Pad. """ - messages = [generate_message() for _ in range(9)] - messages = '\n\n'.join(f'
MESSAGE {i}
{j}
' for i, j in enumerate(messages, 1)) - return PAGE_HTML.format(messages=messages) - - -def generate_pdf(): - # Use False to return bytes, rather than saving to file. - return pdfkit.from_string(generate_html(), False) + messages = '\n\n'.join(f'MESSAGE {i}\n{generate_message()}' for i in range(1, 8)) + with temporary_directory_path() as d: + tex_path = d / 'otp.tex' + tex_path.write_text(ONE_TIME_PAD_TEX % messages) + cmd = ('pdflatex', tex_path) + try: + subprocess.check_output(cmd, cwd=d) + # Output path is chosen by `pdflatex`. + pdf_path = tex_path.with_suffix('.pdf') + contents = pdf_path.read_bytes() + except Exception as e: + logger.fatal(f'Failed to generate One Time Pad PDF!', exc_info=e) + raise + # Files are cleaned up. + assert not d.is_dir() + return contents def validate_message(otp, error_class): diff --git a/modules/otp/test/test_otp.py b/modules/otp/test/test_otp.py index 6f694e25..42a247e6 100644 --- a/modules/otp/test/test_otp.py +++ b/modules/otp/test/test_otp.py @@ -1,6 +1,6 @@ import pytest -from modules.otp.common import format_message, encrypt_otp, decrypt_otp +from modules.otp.common import format_message, encrypt_otp, decrypt_otp, generate_pdf @pytest.mark.parametrize( @@ -82,8 +82,22 @@ def test_decrypt_otp(otp, ciphertext, plaintext): 'ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890', 'ABCDE FGHIJ KLMNO PQRST UVWXY\nZ0123 45678 90', ), + ( + ''.join('A' for _ in range(94)), + 'AAAAA AAAAA AAAAA AAAAA AAAAA\n' + 'AAAAA AAAAA AAAAA AAAAA AAAAA\n' + 'AAAAA AAAAA AAAAA AAAAA AAAAA\n' + 'AAAAA AAAAA AAAAA AAAA' + ), ] ) def test_format_message(message, expected): result = format_message(message) assert result == expected + + +def test_latex_pdf(): + """ + Generate a OTP PDF. + """ + assert generate_pdf() diff --git a/modules/videos/api.py b/modules/videos/api.py index 82af9f0b..9a8b9dcb 100644 --- a/modules/videos/api.py +++ b/modules/videos/api.py @@ -130,6 +130,6 @@ async def statistics(_: Request): return json_response(ret, HTTPStatus.OK) -@before_startup -def startup_spread_channel_downloads(): - channel_lib.spread_channel_downloads() +# @before_startup +# def startup_spread_channel_downloads(): +# channel_lib.spread_channel_downloads() diff --git a/requirements.txt b/requirements.txt index b07e20c4..11445447 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ Sanic-Cors==1.0.1 sanic-openapi==0.6.2 sanic-testing==0.7.0 sanic==21.9.1 +selenium==4.1.0 SQLAlchemy==1.3.22 srt==3.4.1 websockets==10.1 diff --git a/wrolpi/common.py b/wrolpi/common.py index 0e2c9261..a228f5a8 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1,5 +1,6 @@ import asyncio import collections +import contextlib import inspect import json import logging @@ -7,6 +8,7 @@ import pathlib import queue import re import string +import tempfile from datetime import datetime, date from functools import wraps from itertools import islice @@ -506,3 +508,9 @@ def api_param_limiter(maximum: int, default: int = 20) -> callable: return min(int(i), maximum) return limiter_ + + +@contextlib.contextmanager +def temporary_directory_path(*a, **kw): + with tempfile.TemporaryDirectory(*a, **kw) as d: + yield pathlib.Path(d)