From 9194e47e764a85765e0b067ec67b083a20faf240 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 15 Jan 2022 19:34:12 -0700 Subject: [PATCH 01/13] Customizing nginx error page to help user start wrolpi services. --- 50x.html | 16 ++++++++++++++++ install.sh | 6 ++++++ nginx.conf | 9 +++++++-- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 50x.html diff --git a/50x.html b/50x.html new file mode 100644 index 00000000..7b057994 --- /dev/null +++ b/50x.html @@ -0,0 +1,16 @@ + + + +

Error!

+ +

nginx could not find WROLPi! Are your services running?

+ +

sudo systemctl status wrolpi-app

+

sudo systemctl status wrolpi-api

+ +

If either of those services are not running, start it:

+ +

sudo systemctl start wrolpi-app

+

sudo systemctl start wrolpi-api

+ + diff --git a/install.sh b/install.sh index c492ed58..6bbc58f4 100755 --- a/install.sh +++ b/install.sh @@ -89,8 +89,12 @@ sudo -u postgres psql -c '\l' | grep wrolpi || ( # Install the WROLPi nginx config over the default nginx config. cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf +cp /opt/wrolpi/50x.html /var/www/50x.html /usr/sbin/nginx -s reload +# Create the media directory. This should be mounted by the maintainer. +[ -d /media/wrolpi ] || mkdir /media/wrolpi + # Create the WROLPi user grep wrolpi /etc/passwd || useradd -d /opt/wrolpi wrolpi -s "$(command -v bash)" chown -R wrolpi:wrolpi /opt/wrolpi @@ -121,6 +125,8 @@ echo " WROLPi has successfully been installed! +Mount your external hard drive to /media/wrolpi if you have one. + Start the WROLPi services using: # sudo systemctl start wrolpi.target diff --git a/nginx.conf b/nginx.conf index 62d6f5e7..c824520a 100644 --- a/nginx.conf +++ b/nginx.conf @@ -12,9 +12,14 @@ http { charset utf-8; include mime.types; + error_page 502 /error/50x.html; + error_page 503 /error/50x.html; + error_page 504 /error/50x.html; + server { - location /favicon.ico { - alias /opt/wrolpi/icon.ico; + location ^~ /error/ { + internal; + alias /var/www/; } location /apple-touch-icon.png { From b5ea4fe3de8831071ad4b5381e26222a9105ab51 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 15 Jan 2022 19:52:55 -0700 Subject: [PATCH 02/13] Forcing reset of branch when checking out git branch. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 6bbc58f4..c8894b16 100755 --- a/install.sh +++ b/install.sh @@ -56,7 +56,7 @@ npm install -g 'git+https://github.com/pirate/readability-extractor' # Get the latest WROLPi code git --version git clone https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi || : -(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git pull origin "${BRANCH}" --rebase) || exit 3 +(cd /opt/wrolpi && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 3 # Setup the virtual environment that main.py expects pip3 --version || ( From e0b59667175c2a9a5fb63050e63b6116d2a66709 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 15 Jan 2022 19:58:38 -0700 Subject: [PATCH 03/13] Creating videos and archives directory when they don't exist. --- modules/archive/lib.py | 5 ++++- modules/videos/downloader.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index fc2eecff..edec524e 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -26,7 +26,10 @@ ARCHIVE_SERVICE = 'http://archive:8080' def get_archive_directory() -> pathlib.Path: - return get_media_directory() / 'archive' + archive_directory = get_media_directory() / 'archive' + if not archive_directory.is_dir(): + archive_directory.mkdir() + return archive_directory def get_domain_directory(url: str) -> pathlib.Path: diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 56d74c71..7fb82f48 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -150,7 +150,7 @@ class VideoDownloader(Downloader, ABC): out_dir = get_no_channel_directory() if channel: out_dir = channel.directory.path - out_dir.mkdir(exist_ok=True) + out_dir.mkdir(exist_ok=True, parents=True) try: video_path, entry = self.prepare_filename(url, out_dir) From fac3a8eba8bcc890ae13ad21f8f7d1d854e68b45 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 15 Jan 2022 19:59:36 -0700 Subject: [PATCH 04/13] Fetching before resetting. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index c8894b16..b30e7173 100755 --- a/install.sh +++ b/install.sh @@ -56,7 +56,7 @@ npm install -g 'git+https://github.com/pirate/readability-extractor' # Get the latest WROLPi code git --version git clone https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi || : -(cd /opt/wrolpi && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 3 +(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 3 # Setup the virtual environment that main.py expects pip3 --version || ( From c00bc98755c5bb0d83c4c3e291c5eefa522da69a Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 15 Jan 2022 20:00:36 -0700 Subject: [PATCH 05/13] Fixing ownership of mount directory. --- install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install.sh b/install.sh index b30e7173..260ecd6b 100755 --- a/install.sh +++ b/install.sh @@ -94,6 +94,7 @@ cp /opt/wrolpi/50x.html /var/www/50x.html # Create the media directory. This should be mounted by the maintainer. [ -d /media/wrolpi ] || mkdir /media/wrolpi +chown -R wrolpi:wrolpi /media/wrolpi # Create the WROLPi user grep wrolpi /etc/passwd || useradd -d /opt/wrolpi wrolpi -s "$(command -v bash)" From ca96bce6b7cf76a0754e46e5a2d070155cc89f50 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 15 Jan 2022 20:07:38 -0700 Subject: [PATCH 06/13] Finding yt-dlp bin on startup. --- modules/videos/downloader.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 7fb82f48..1987aef0 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -20,7 +20,7 @@ from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownload from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel from .common import load_downloader_config, apply_info_json, get_channel_source_id -from .lib import upsert_video, _refresh_videos, refresh_channel_videos +from .lib import upsert_video, refresh_channel_videos from .models import Video, Channel from .video_url_resolver import video_url_resolver @@ -94,6 +94,13 @@ class ChannelDownloader(Downloader, ABC): return True +YT_DLP_BIN = pathlib.Path('/usr/local/bin/yt-dlp') # Location in docker container +if not YT_DLP_BIN.is_file(): + YT_DLP_BIN = pathlib.Path('/opt/wrolpi/venv/bin/yt-dlp') # Use virtual environment location +if not YT_DLP_BIN.is_file(): + logger.error('COULD NOT FIND YT-DLP!!!') + + class VideoDownloader(Downloader, ABC): """ Download a single video. Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`. @@ -157,7 +164,7 @@ class VideoDownloader(Downloader, ABC): # Do the real download. file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s' cmd = [ - 'yt-dlp', + str(YT_DLP_BIN), '-cw', # Continue downloads, do not clobber existing files. '-f', PREFERRED_VIDEO_FORMAT, '--write-subs', From ba59a2c75e157554450d9d97d1cd1da4229f0751 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 16 Jan 2022 09:22:03 -0700 Subject: [PATCH 07/13] Creating user before media directory. --- install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 260ecd6b..e2e50d78 100755 --- a/install.sh +++ b/install.sh @@ -92,14 +92,14 @@ cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf cp /opt/wrolpi/50x.html /var/www/50x.html /usr/sbin/nginx -s reload -# Create the media directory. This should be mounted by the maintainer. -[ -d /media/wrolpi ] || mkdir /media/wrolpi -chown -R wrolpi:wrolpi /media/wrolpi - # Create the WROLPi user grep wrolpi /etc/passwd || useradd -d /opt/wrolpi wrolpi -s "$(command -v bash)" chown -R wrolpi:wrolpi /opt/wrolpi +# Create the media directory. This should be mounted by the maintainer. +[ -d /media/wrolpi ] || mkdir /media/wrolpi +chown -R wrolpi:wrolpi /media/wrolpi + # Install the systemd services cp /opt/wrolpi/systemd/wrolpi-api.service /etc/systemd/system/ cp /opt/wrolpi/systemd/wrolpi-app.service /etc/systemd/system/ From a70e0498237fca982b9fd935a40842cf33b27c6c Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 16 Jan 2022 09:28:22 -0700 Subject: [PATCH 08/13] Creating channel directory if it does not exist. --- modules/videos/downloader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 1987aef0..4d1f9537 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -262,6 +262,8 @@ def get_or_create_channel(source_id: str = None, link: str = None, url: str = No pass channel_directory = get_media_directory() / f'videos/{name}' + if not channel_directory.is_dir(): + channel_directory.mkdir(parents=True) data = dict( name=name, url=url, From 89e3edb1578fa6888b2a39960d16c8a916c62fc9 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 16 Jan 2022 10:17:06 -0700 Subject: [PATCH 09/13] Creating functions to unify finding/making video directories. --- modules/videos/channel/test/test_api.py | 3 +-- modules/videos/common.py | 22 +++++++++++++++++++++- modules/videos/downloader.py | 13 +++++-------- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index b859e62a..a610bb92 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -11,6 +11,7 @@ import pytest from modules.videos.api import refresh_queue from modules.videos.channel.lib import spread_channel_downloads, delete_channel +from modules.videos.common import get_no_channel_directory from modules.videos.lib import upsert_video from modules.videos.models import Channel, Video from modules.videos.test.common import create_channel_structure @@ -139,9 +140,7 @@ class TestVideoAPI(TestAPI): session.commit() # The special NO CHANNEL directory. - from modules.videos.downloader import get_no_channel_directory no_channel_path = get_no_channel_directory() - no_channel_path.mkdir() vid3 = no_channel_path / 'some video.mp4' vid3.touch() no_channel_subdir = no_channel_path / 'subdir' diff --git a/modules/videos/common.py b/modules/videos/common.py index b558628d..6dcdcec8 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -56,6 +56,26 @@ class ConfigError(Exception): pass +def get_videos_directory() -> pathlib.Path: + """ + Get the "videos" directory in the media directory. Make it if it does not exist. + """ + directory = get_media_directory() / 'videos' + if not directory.is_dir(): + directory.mkdir(parents=True) + return directory + + +def get_no_channel_directory() -> pathlib.Path: + """ + Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist. + """ + directory = get_videos_directory() / 'NO CHANNEL' + if not directory.is_dir(): + directory.mkdir(parents=True) + return directory + + @before_startup def import_videos_config(): """Import channel settings to the DB. Existing channels will be updated.""" @@ -291,7 +311,7 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]: path = str(path) ignored_directories = { - str(get_media_directory() / 'videos/NO CHANNEL'), + str(get_no_channel_directory()), } if os.path.isdir(path): diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 4d1f9537..8b8138d8 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -11,15 +11,15 @@ from yt_dlp import YoutubeDL from yt_dlp.extractor import YoutubeTabIE from yt_dlp.utils import UnsupportedError, DownloadError -from wrolpi.common import logger, sanitize_link, extract_domain, \ - get_media_directory +from wrolpi.common import logger, sanitize_link, extract_domain from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs from wrolpi.downloader import Downloader, Download from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel -from .common import load_downloader_config, apply_info_json, get_channel_source_id +from .common import load_downloader_config, apply_info_json, get_channel_source_id, get_no_channel_directory, \ + get_videos_directory from .lib import upsert_video, refresh_channel_videos from .models import Video, Channel from .video_url_resolver import video_url_resolver @@ -40,10 +40,6 @@ PREFERRED_VIDEO_EXTENSION = 'mp4' PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p' -def get_no_channel_directory(): - return get_media_directory() / 'videos/NO CHANNEL' - - class ChannelDownloader(Downloader, ABC): """ Handling downloading of all Videos in a Channel. @@ -261,7 +257,8 @@ def get_or_create_channel(source_id: str = None, link: str = None, url: str = No except UnknownChannel: pass - channel_directory = get_media_directory() / f'videos/{name}' + # Channel does not exist. Create one in the video directory. + channel_directory = get_videos_directory() / name if not channel_directory.is_dir(): channel_directory.mkdir(parents=True) data = dict( From 1d9fe2f6499258f46e8551702598c5fc8e8a45f9 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 16 Jan 2022 10:27:24 -0700 Subject: [PATCH 10/13] Setting channel id rather than channel because the channel may be in a different session. --- modules/videos/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index ad94dbce..e05cacad 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -439,7 +439,7 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = if channel and not str(video_path).startswith(str(channel.directory.path)): raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}') - video.channel = channel + video.channel_id = channel.id video.idempotency = idempotency try: From 6f58a2b5639eb7fc9d45122c067b2f54b040ab0a Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 16 Jan 2022 13:35:18 -0700 Subject: [PATCH 11/13] Chrome attempts to screenshot in the current directory, but if the current directory is not a real user directory, the screenshot fail.s --- modules/archive/lib.py | 20 +++++++++++-------- wrolpi/common.py | 40 ++++++++++++++++++++++++++++++++++++++ wrolpi/test/test_common.py | 33 ++++++++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index edec524e..adc3177d 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -14,7 +14,7 @@ from selenium import webdriver from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive -from wrolpi.common import get_media_directory, logger, chunks, extract_domain +from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir from wrolpi.dates import now, strptime_ms, strftime_ms from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive @@ -144,6 +144,7 @@ def local_screenshot(url: str) -> bytes: # Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page. options = webdriver.ChromeOptions() options.add_argument('headless') + options.add_argument('disable-gpu') options.add_argument('window-size=1280x720') driver = webdriver.Chrome(chrome_options=options) @@ -177,13 +178,16 @@ def local_archive(url: str): """ Perform an archive of the provided URL using local resources (without the Archive docker container). """ - singlefile = local_singlefile(url) - with tempfile.NamedTemporaryFile('wb') as fh: - fh.write(singlefile) - readability = local_extract_readability(fh.name, url) - screenshot = local_screenshot(url) - singlefile = singlefile.decode() - return singlefile, readability, screenshot + # Archives must be performed in the wrolpi home directory because chrome saves the screenshot there, and will + # raise an error if global $HOME is not a real user directory. :( + with chdir('/home/wrolpi', with_home=True): + singlefile = local_singlefile(url) + with tempfile.NamedTemporaryFile('wb') as fh: + fh.write(singlefile) + readability = local_extract_readability(fh.name, url) + screenshot = local_screenshot(url) + singlefile = singlefile.decode() + return singlefile, readability, screenshot def do_archive(url: str) -> Archive: diff --git a/wrolpi/common.py b/wrolpi/common.py index c4f005d6..2351cb22 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -4,6 +4,7 @@ import contextlib import inspect import json import logging +import os import pathlib import queue import re @@ -526,3 +527,42 @@ def partition(pred, iterable): """ t1, t2 = tee(iterable) return list(filter(pred, t2)), list(filterfalse(pred, t1)) + + +@contextlib.contextmanager +def chdir(directory: Union[pathlib.Path, str, None] = None, with_home: bool = False): + """ + Change the Python working directory in this context. If no directory is passed, a TemporaryDirectory will be used. + + This also changes the $HOME environment variable to match. + """ + home_exists = 'HOME' in os.environ + home = os.environ.get('HOME') + cwd = os.getcwd() + if directory is None: + with tempfile.TemporaryDirectory() as d: + try: + os.chdir(d) + if with_home: + os.environ['HOME'] = d + yield + finally: + os.chdir(cwd) + if home_exists: + os.environ['HOME'] = home + else: + del os.environ['HOME'] + return + + try: + os.chdir(directory) + if with_home: + os.environ['HOME'] = str(directory) + yield + finally: + os.chdir(cwd) + if home_exists: + os.environ['HOME'] = home + else: + del os.environ['HOME'] + return diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index a00a5764..c9c4a103 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1,10 +1,13 @@ +import os +import pathlib +import tempfile import unittest from datetime import date, datetime import pytest from sanic_openapi import doc -from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter +from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir from wrolpi.dates import set_timezone, now from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone from wrolpi.schema import validate_data @@ -315,3 +318,31 @@ class TestCommon(unittest.TestCase): def test_api_param_limiter(i, expected): limiter = api_param_limiter(100) # should never return an integer greater than 100. assert limiter(i) == expected + + +def test_chdir(): + """ + The current working directory can be changed temporarily using the `chdir` context manager. + """ + original = os.getcwd() + home = os.environ.get('HOME') + assert home + + with chdir(): + assert os.getcwd() != original + assert str(os.getcwd()).startswith('/tmp') + assert os.environ['HOME'] != os.getcwd() + # Replace $HOME + with chdir(with_home=True): + assert os.getcwd() != original + assert str(os.getcwd()).startswith('/tmp') + assert os.environ['HOME'] == os.getcwd() + + with tempfile.TemporaryDirectory() as d: + # Without replacing $HOME + with chdir(pathlib.Path(d), with_home=True): + assert os.getcwd() == d + assert os.environ['HOME'] == os.getcwd() + with chdir(pathlib.Path(d)): + assert os.getcwd() == d + assert os.environ['HOME'] != os.getcwd() From f305d7912fc76144e5703def80f8b01c5de091b9 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 16 Jan 2022 13:37:44 -0700 Subject: [PATCH 12/13] channel may be None. --- modules/videos/lib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index e05cacad..08439353 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -439,7 +439,8 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = if channel and not str(video_path).startswith(str(channel.directory.path)): raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}') - video.channel_id = channel.id + if channel: + video.channel_id = channel.id video.idempotency = idempotency try: From 3ee34fef42e28c0e20d568954beb6dbb91602c3f Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 16 Jan 2022 13:38:51 -0700 Subject: [PATCH 13/13] Changing wrolpi user home directory to reflect what Chrome requires. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index e2e50d78..e0056a48 100755 --- a/install.sh +++ b/install.sh @@ -93,7 +93,7 @@ cp /opt/wrolpi/50x.html /var/www/50x.html /usr/sbin/nginx -s reload # Create the WROLPi user -grep wrolpi /etc/passwd || useradd -d /opt/wrolpi wrolpi -s "$(command -v bash)" +grep wrolpi /etc/passwd || useradd -d /home/wrolpi wrolpi -s "$(command -v bash)" chown -R wrolpi:wrolpi /opt/wrolpi # Create the media directory. This should be mounted by the maintainer.