Feature/raspberry pi archive (#72)

* Converting OTP PDF generation to Latex to avoid installing wkhtmltopdf.

* Creating temporary_directory_path to simplify creating a TemporaryDirectory of type Path.

* Removing author

* Linting

* Installing texlive-latex-extra for more readable font.

* Raspberry pi has to archive locally.

Signed-off-by: Roland <roland@learningselfreliance.com>

docker/api/Dockerfile

@@ -4,7 +4,9 @@ WORKDIR /opt/wrolpi
 
 # Update dependencies for the services install.
 RUN apt update
-RUN apt install -y ffmpeg wkhtmltopdf
+# ffmpeg for Videos and Latex for OTP PDF.
+RUN apt-get install -y ffmpeg texlive-latex-base texlive-latex-extra
+RUN pdflatex --version
 
 # Install WROLPi
 COPY main.py /opt/wrolpi/

install.sh

@@ -47,7 +47,11 @@ curl -fsSL https://deb.nodesource.com/setup_14.x | bash -
 # Install dependencies
 apt install -y git apt-transport-https ca-certificates curl gnupg-agent gcc libpq-dev software-properties-common \
   postgresql-12 nginx-full nginx-doc python3.8-minimal python3.8-dev python3.8-doc python3.8-venv \
-  ffmpeg wkhtmltopdf hostapd nodejs yarn
+  ffmpeg hostapd nodejs yarn texlive-latex-base texlive-latex-extra chromium-browser chromium-chromedriver
+
+# Install Archiving tools.
+npm install -g "gildas-lormeau/SingleFile#master"
+npm install -g 'git+https://github.com/pirate/readability-extractor'
 
 # Get the latest WROLPi code
 git --version

main.py

@@ -121,6 +121,7 @@ def periodic_downloads(app: Sanic, loop):
     if wrol_mode_enabled():
         logger.warning(f'Not starting download worker because WROL Mode is enabled.')
         return
+    return
     logger.info('Starting download manager.')
 
     # Set all downloads to new.

modules/archive/lib.py

@@ -3,11 +3,14 @@ import gzip
 import json
 import pathlib
 import re
+import subprocess
+import tempfile
 from itertools import groupby
 from typing import Iterator
 
 import requests
 from bs4 import BeautifulSoup
+from selenium import webdriver
 from sqlalchemy.orm import Session
 
 from modules.archive.models import Domain, Archive
@@ -15,6 +18,7 @@ from wrolpi.common import get_media_directory, logger, chunks, extract_domain
 from wrolpi.dates import now, strptime_ms, strftime_ms
 from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
 from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
+from wrolpi.vars import DOCKERIZED
 
 logger = logger.getChild(__name__)
 
@@ -69,6 +73,8 @@ def request_archive(url: str):
     """
     Send a request to the archive service to archive the URL.
     """
+    logger.info(f'Sending archive request to archive service: {url}')
+
     data = {'url': url}
     try:
         resp = requests.post(f'{ARCHIVE_SERVICE}/json', json=data, timeout=ARCHIVE_TIMEOUT)
@@ -107,15 +113,82 @@ def request_archive(url: str):
     return singlefile, readability, screenshot
 
 
+def local_singlefile(url: str):
+    single_file_path = pathlib.Path('/usr/bin/single-file')
+    if not single_file_path.is_file():
+        raise FileNotFoundError(f'single-file not found at {single_file_path}')
+
+    cmd = (str(single_file_path),
+           url,
+           '--browser-executable-path', '/usr/bin/chromium-browser',
+           '--browser-args', '["--no-sandbox"]',
+           '--dump-content')
+    logger.debug(f'archive cmd: {cmd}')
+    output = subprocess.check_output(cmd, timeout=60 * 3)
+    logger.debug(f'done archiving for {url}')
+    return output
+
+
+def local_screenshot(url: str) -> bytes:
+    logger.info(f'Screenshot: {url}')
+
+    # Set Chromium to headless.  Use a wide window size so that screenshot will be the "desktop" version of the page.
+    options = webdriver.ChromeOptions()
+    options.add_argument('headless')
+    options.add_argument('window-size=1280x720')
+
+    driver = webdriver.Chrome(chrome_options=options)
+    try:
+        driver.get(url)
+        png = driver.get_screenshot_as_png()
+    except Exception as e:
+        logger.warning(f'Failed to screenshot {url}', exc_info=e)
+        return b''
+    return png
+
+
+def local_extract_readability(path: str, url: str) -> dict:
+    logger.info(f'readability for {url}')
+    readability_path = pathlib.Path('/usr/bin/readability-extractor')
+    if not readability_path.is_file():
+        raise FileNotFoundError(f'Readability extractor not found at {readability_path}')
+
+    cmd = (readability_path, path, url)
+    logger.debug(f'readability cmd: {cmd}')
+    output = subprocess.check_output(cmd, timeout=60 * 3)
+    output = json.loads(output)
+    logger.debug(f'done readability for {url}')
+    return output
+
+
+def local_archive(url: str):
+    """
+    Perform an archive of the provided URL using local resources (without the Archive docker container).
+    """
+    singlefile = local_singlefile(url)
+    with tempfile.NamedTemporaryFile('wb') as fh:
+        fh.write(singlefile)
+        readability = local_extract_readability(fh.name, url)
+    screenshot = local_screenshot(url)
+    singlefile = singlefile.decode()
+    return singlefile, readability, screenshot
+
+
 def do_archive(url: str) -> Archive:
     """
     Perform the real archive request to the archiving service.  Store the resulting data into files.  Create an Archive
     record in the DB.  Create Domain/URL if missing.
     """
+    logger.info(f'Archiving {url}')
     singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
         get_new_archive_files(url)
 
-    singlefile, readability, screenshot = request_archive(url)
+    if DOCKERIZED:
+        # Perform the archive in the Archive docker container.  (Typically in the development environment).
+        singlefile, readability, screenshot = request_archive(url)
+    else:
+        # Perform the archive using locally installed executables.
+        singlefile, readability, screenshot = local_archive(url)
 
     # Store the single-file HTML in its own file.
     with singlefile_path.open('wt') as fh:

modules/otp/api.py

@@ -7,7 +7,7 @@ from sanic.request import Request
 
 from wrolpi.root_api import get_blueprint
 from wrolpi.schema import validate_doc
-from .common import generate_html, generate_pdf, encrypt_otp, decrypt_otp
+from .common import generate_pdf, encrypt_otp, decrypt_otp
 from .schema import EncryptOTPRequest, DecryptOTPRequest
 
 bp = get_blueprint('OTP', '/api/otp')
@@ -33,11 +33,6 @@ async def post_decrypt_otp(_: Request, data: dict):
     return response.json(data)
 
 
-@bp.get('/html')
-async def get_new_otp(_: Request):
-    return response.html(generate_html())
-
-
 @bp.get('/pdf')
 async def get_new_otp_pdf(_: Request):
     random_name = ''.join(random.choice(ascii_letters) for _ in range(8))

modules/otp/common.py

@@ -1,13 +1,15 @@
 import secrets
+import subprocess
 from functools import partial
 from typing import Dict
 
-import pdfkit
-
-from wrolpi.common import chunks, remove_whitespace
+from wrolpi.common import chunks, remove_whitespace, logger, temporary_directory_path
 from wrolpi.errors import InvalidOTP, InvalidPlaintext, InvalidCiphertext
 
 # These are the only characters we support.
+
+logger = logger.getChild(__name__)
+
 OTP_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
 # The number of characters in each group, separated by spaces.
 GROUP_SIZE = 5
@@ -36,48 +38,57 @@ def generate_message():
     Create a OTP message.  Format it for ease of use.
     """
     # Use 20 messages per line because this page will be printed.
-    return format_message(''.join(generate_char() for _ in range(400)), 20)
+    return format_message(''.join(generate_char() for _ in range(400)), 16)
 
 
-PAGE_HTML = '''
-<html>
-<title>One Time Pad - Unique, just for you</title>
-<style>
-/* Remove decoration from links so they are readable when printed */
-a {{ text-decoration: none; }}  // escaped for python
-</style>
-<body>
-{messages}
+ONE_TIME_PAD_TEX = r'''
+\documentclass{article}
+\usepackage{hyperref}
+\usepackage[margin=0.3in]{geometry}
+\usepackage{mathptmx}
 
+\title{One Time Pad}
 
-Print this page and distribute the copies (along with the <a href="https://lrnsr.co/aY6m">One Time Pad Cheat Sheet
-https://lrnsr.co/aY6m</a>) to all members of your group that you trust to receive your encrypted messages.  Every
-person must have their OWN copy of this "One Time Pad" to encrypt and decrypt messages.
-<br>
-<b>Use each message ONLY ONCE.</b>  Cut off and burn each message from this paper as it is used.
-<br>
-If you want more One Time Pads, simply <a href=".">go here to refresh the page:
-https://learningselfreliance.com/one_time_pad</a>.  The server will generate a unique page just for you. This page
-is not stored on the server, and cannot be retrieved once you close this window!
-<br>
-To learn how to use this page, please visit: <a href="https://lrnsr.co/H7Za">https://lrnsr.co/H7Za</a>
-</body>
-</html>
+\begin{document}
+
+\begin{verbatim}
+%s
+\end{verbatim}
+
+Print this page and distribute the copies (along with the \href{https://lrnsr.co/aY6m}{One Time Pad Cheat Sheet
+https://lrnsr.co/aY6m}) to all members of your group that you trust to receive your encrypted messages.  Every person
+must have their OWN copy of this “One Time Pad” to encrypt and decrypt messages.
+\\
+
+\textbf{Use each message ONLY ONCE}.  Cut off and burn each message from this paper as it is used.
+\\
+
+To learn how to use this page, please visit: \href{https://lrnsr.co/H7Za}{https://lrnsr.co/H7Za}
+
+\end{document}
 '''
 
 
-def generate_html() -> str:
+def generate_pdf() -> bytes:
     """
-    Create an HTML One-Time Pad page.   This page will have instructions on how to use the OTP.
+    Create a PDF One Time Pad.
     """
-    messages = [generate_message() for _ in range(9)]
-    messages = '\n\n'.join(f'<pre>MESSAGE {i}</pre><pre>{j}</pre>' for i, j in enumerate(messages, 1))
-    return PAGE_HTML.format(messages=messages)
-
-
-def generate_pdf():
-    # Use False to return bytes, rather than saving to file.
-    return pdfkit.from_string(generate_html(), False)
+    messages = '\n\n'.join(f'MESSAGE {i}\n{generate_message()}' for i in range(1, 8))
+    with temporary_directory_path() as d:
+        tex_path = d / 'otp.tex'
+        tex_path.write_text(ONE_TIME_PAD_TEX % messages)
+        cmd = ('pdflatex', tex_path)
+        try:
+            subprocess.check_output(cmd, cwd=d)
+            # Output path is chosen by `pdflatex`.
+            pdf_path = tex_path.with_suffix('.pdf')
+            contents = pdf_path.read_bytes()
+        except Exception as e:
+            logger.fatal(f'Failed to generate One Time Pad PDF!', exc_info=e)
+            raise
+    # Files are cleaned up.
+    assert not d.is_dir()
+    return contents
 
 
 def validate_message(otp, error_class):

modules/otp/test/test_otp.py

@@ -1,6 +1,6 @@
 import pytest
 
-from modules.otp.common import format_message, encrypt_otp, decrypt_otp
+from modules.otp.common import format_message, encrypt_otp, decrypt_otp, generate_pdf
 
 
 @pytest.mark.parametrize(
@@ -82,8 +82,22 @@ def test_decrypt_otp(otp, ciphertext, plaintext):
                 'ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890',
                 'ABCDE FGHIJ KLMNO PQRST UVWXY\nZ0123 45678 90',
         ),
+        (
+                ''.join('A' for _ in range(94)),
+                'AAAAA AAAAA AAAAA AAAAA AAAAA\n'
+                'AAAAA AAAAA AAAAA AAAAA AAAAA\n'
+                'AAAAA AAAAA AAAAA AAAAA AAAAA\n'
+                'AAAAA AAAAA AAAAA AAAA'
+        ),
     ]
 )
 def test_format_message(message, expected):
     result = format_message(message)
     assert result == expected
+
+
+def test_latex_pdf():
+    """
+    Generate a OTP PDF.
+    """
+    assert generate_pdf()

modules/videos/api.py

@@ -130,6 +130,6 @@ async def statistics(_: Request):
     return json_response(ret, HTTPStatus.OK)
 
 
-@before_startup
-def startup_spread_channel_downloads():
-    channel_lib.spread_channel_downloads()
+# @before_startup
+# def startup_spread_channel_downloads():
+#     channel_lib.spread_channel_downloads()

requirements.txt

@@ -13,6 +13,7 @@ Sanic-Cors==1.0.1
 sanic-openapi==0.6.2
 sanic-testing==0.7.0
 sanic==21.9.1
+selenium==4.1.0
 SQLAlchemy==1.3.22
 srt==3.4.1
 websockets==10.1

wrolpi/common.py

@@ -1,5 +1,6 @@
 import asyncio
 import collections
+import contextlib
 import inspect
 import json
 import logging
@@ -7,6 +8,7 @@ import pathlib
 import queue
 import re
 import string
+import tempfile
 from datetime import datetime, date
 from functools import wraps
 from itertools import islice
@@ -506,3 +508,9 @@ def api_param_limiter(maximum: int, default: int = 20) -> callable:
         return min(int(i), maximum)
 
     return limiter_
+
+
+@contextlib.contextmanager
+def temporary_directory_path(*a, **kw):
+    with tempfile.TemporaryDirectory(*a, **kw) as d:
+        yield pathlib.Path(d)