diff --git a/50x.html b/50x.html new file mode 100644 index 00000000..7b057994 --- /dev/null +++ b/50x.html @@ -0,0 +1,16 @@ + +
+ +nginx could not find WROLPi! Are your services running?
+ +sudo systemctl status wrolpi-app
+sudo systemctl status wrolpi-api
+ +If either of those services are not running, start it:
+ +sudo systemctl start wrolpi-app
+sudo systemctl start wrolpi-api
+ + diff --git a/install.sh b/install.sh index c492ed58..e0056a48 100755 --- a/install.sh +++ b/install.sh @@ -56,7 +56,7 @@ npm install -g 'git+https://github.com/pirate/readability-extractor' # Get the latest WROLPi code git --version git clone https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi || : -(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git pull origin "${BRANCH}" --rebase) || exit 3 +(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 3 # Setup the virtual environment that main.py expects pip3 --version || ( @@ -89,12 +89,17 @@ sudo -u postgres psql -c '\l' | grep wrolpi || ( # Install the WROLPi nginx config over the default nginx config. cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf +cp /opt/wrolpi/50x.html /var/www/50x.html /usr/sbin/nginx -s reload # Create the WROLPi user -grep wrolpi /etc/passwd || useradd -d /opt/wrolpi wrolpi -s "$(command -v bash)" +grep wrolpi /etc/passwd || useradd -d /home/wrolpi wrolpi -s "$(command -v bash)" chown -R wrolpi:wrolpi /opt/wrolpi +# Create the media directory. This should be mounted by the maintainer. +[ -d /media/wrolpi ] || mkdir /media/wrolpi +chown -R wrolpi:wrolpi /media/wrolpi + # Install the systemd services cp /opt/wrolpi/systemd/wrolpi-api.service /etc/systemd/system/ cp /opt/wrolpi/systemd/wrolpi-app.service /etc/systemd/system/ @@ -121,6 +126,8 @@ echo " WROLPi has successfully been installed! +Mount your external hard drive to /media/wrolpi if you have one. + Start the WROLPi services using: # sudo systemctl start wrolpi.target diff --git a/modules/archive/lib.py b/modules/archive/lib.py index fc2eecff..adc3177d 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -14,7 +14,7 @@ from selenium import webdriver from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive -from wrolpi.common import get_media_directory, logger, chunks, extract_domain +from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir from wrolpi.dates import now, strptime_ms, strftime_ms from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive @@ -26,7 +26,10 @@ ARCHIVE_SERVICE = 'http://archive:8080' def get_archive_directory() -> pathlib.Path: - return get_media_directory() / 'archive' + archive_directory = get_media_directory() / 'archive' + if not archive_directory.is_dir(): + archive_directory.mkdir() + return archive_directory def get_domain_directory(url: str) -> pathlib.Path: @@ -141,6 +144,7 @@ def local_screenshot(url: str) -> bytes: # Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page. options = webdriver.ChromeOptions() options.add_argument('headless') + options.add_argument('disable-gpu') options.add_argument('window-size=1280x720') driver = webdriver.Chrome(chrome_options=options) @@ -174,13 +178,16 @@ def local_archive(url: str): """ Perform an archive of the provided URL using local resources (without the Archive docker container). """ - singlefile = local_singlefile(url) - with tempfile.NamedTemporaryFile('wb') as fh: - fh.write(singlefile) - readability = local_extract_readability(fh.name, url) - screenshot = local_screenshot(url) - singlefile = singlefile.decode() - return singlefile, readability, screenshot + # Archives must be performed in the wrolpi home directory because chrome saves the screenshot there, and will + # raise an error if global $HOME is not a real user directory. :( + with chdir('/home/wrolpi', with_home=True): + singlefile = local_singlefile(url) + with tempfile.NamedTemporaryFile('wb') as fh: + fh.write(singlefile) + readability = local_extract_readability(fh.name, url) + screenshot = local_screenshot(url) + singlefile = singlefile.decode() + return singlefile, readability, screenshot def do_archive(url: str) -> Archive: diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index b859e62a..a610bb92 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -11,6 +11,7 @@ import pytest from modules.videos.api import refresh_queue from modules.videos.channel.lib import spread_channel_downloads, delete_channel +from modules.videos.common import get_no_channel_directory from modules.videos.lib import upsert_video from modules.videos.models import Channel, Video from modules.videos.test.common import create_channel_structure @@ -139,9 +140,7 @@ class TestVideoAPI(TestAPI): session.commit() # The special NO CHANNEL directory. - from modules.videos.downloader import get_no_channel_directory no_channel_path = get_no_channel_directory() - no_channel_path.mkdir() vid3 = no_channel_path / 'some video.mp4' vid3.touch() no_channel_subdir = no_channel_path / 'subdir' diff --git a/modules/videos/common.py b/modules/videos/common.py index b558628d..6dcdcec8 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -56,6 +56,26 @@ class ConfigError(Exception): pass +def get_videos_directory() -> pathlib.Path: + """ + Get the "videos" directory in the media directory. Make it if it does not exist. + """ + directory = get_media_directory() / 'videos' + if not directory.is_dir(): + directory.mkdir(parents=True) + return directory + + +def get_no_channel_directory() -> pathlib.Path: + """ + Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist. + """ + directory = get_videos_directory() / 'NO CHANNEL' + if not directory.is_dir(): + directory.mkdir(parents=True) + return directory + + @before_startup def import_videos_config(): """Import channel settings to the DB. Existing channels will be updated.""" @@ -291,7 +311,7 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]: path = str(path) ignored_directories = { - str(get_media_directory() / 'videos/NO CHANNEL'), + str(get_no_channel_directory()), } if os.path.isdir(path): diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 56d74c71..8b8138d8 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -11,16 +11,16 @@ from yt_dlp import YoutubeDL from yt_dlp.extractor import YoutubeTabIE from yt_dlp.utils import UnsupportedError, DownloadError -from wrolpi.common import logger, sanitize_link, extract_domain, \ - get_media_directory +from wrolpi.common import logger, sanitize_link, extract_domain from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs from wrolpi.downloader import Downloader, Download from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel -from .common import load_downloader_config, apply_info_json, get_channel_source_id -from .lib import upsert_video, _refresh_videos, refresh_channel_videos +from .common import load_downloader_config, apply_info_json, get_channel_source_id, get_no_channel_directory, \ + get_videos_directory +from .lib import upsert_video, refresh_channel_videos from .models import Video, Channel from .video_url_resolver import video_url_resolver @@ -40,10 +40,6 @@ PREFERRED_VIDEO_EXTENSION = 'mp4' PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p' -def get_no_channel_directory(): - return get_media_directory() / 'videos/NO CHANNEL' - - class ChannelDownloader(Downloader, ABC): """ Handling downloading of all Videos in a Channel. @@ -94,6 +90,13 @@ class ChannelDownloader(Downloader, ABC): return True +YT_DLP_BIN = pathlib.Path('/usr/local/bin/yt-dlp') # Location in docker container +if not YT_DLP_BIN.is_file(): + YT_DLP_BIN = pathlib.Path('/opt/wrolpi/venv/bin/yt-dlp') # Use virtual environment location +if not YT_DLP_BIN.is_file(): + logger.error('COULD NOT FIND YT-DLP!!!') + + class VideoDownloader(Downloader, ABC): """ Download a single video. Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`. @@ -150,14 +153,14 @@ class VideoDownloader(Downloader, ABC): out_dir = get_no_channel_directory() if channel: out_dir = channel.directory.path - out_dir.mkdir(exist_ok=True) + out_dir.mkdir(exist_ok=True, parents=True) try: video_path, entry = self.prepare_filename(url, out_dir) # Do the real download. file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s' cmd = [ - 'yt-dlp', + str(YT_DLP_BIN), '-cw', # Continue downloads, do not clobber existing files. '-f', PREFERRED_VIDEO_FORMAT, '--write-subs', @@ -254,7 +257,10 @@ def get_or_create_channel(source_id: str = None, link: str = None, url: str = No except UnknownChannel: pass - channel_directory = get_media_directory() / f'videos/{name}' + # Channel does not exist. Create one in the video directory. + channel_directory = get_videos_directory() / name + if not channel_directory.is_dir(): + channel_directory.mkdir(parents=True) data = dict( name=name, url=url, diff --git a/modules/videos/lib.py b/modules/videos/lib.py index ad94dbce..08439353 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -439,7 +439,8 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = if channel and not str(video_path).startswith(str(channel.directory.path)): raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}') - video.channel = channel + if channel: + video.channel_id = channel.id video.idempotency = idempotency try: diff --git a/nginx.conf b/nginx.conf index 62d6f5e7..c824520a 100644 --- a/nginx.conf +++ b/nginx.conf @@ -12,9 +12,14 @@ http { charset utf-8; include mime.types; + error_page 502 /error/50x.html; + error_page 503 /error/50x.html; + error_page 504 /error/50x.html; + server { - location /favicon.ico { - alias /opt/wrolpi/icon.ico; + location ^~ /error/ { + internal; + alias /var/www/; } location /apple-touch-icon.png { diff --git a/wrolpi/common.py b/wrolpi/common.py index c4f005d6..2351cb22 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -4,6 +4,7 @@ import contextlib import inspect import json import logging +import os import pathlib import queue import re @@ -526,3 +527,42 @@ def partition(pred, iterable): """ t1, t2 = tee(iterable) return list(filter(pred, t2)), list(filterfalse(pred, t1)) + + +@contextlib.contextmanager +def chdir(directory: Union[pathlib.Path, str, None] = None, with_home: bool = False): + """ + Change the Python working directory in this context. If no directory is passed, a TemporaryDirectory will be used. + + This also changes the $HOME environment variable to match. + """ + home_exists = 'HOME' in os.environ + home = os.environ.get('HOME') + cwd = os.getcwd() + if directory is None: + with tempfile.TemporaryDirectory() as d: + try: + os.chdir(d) + if with_home: + os.environ['HOME'] = d + yield + finally: + os.chdir(cwd) + if home_exists: + os.environ['HOME'] = home + else: + del os.environ['HOME'] + return + + try: + os.chdir(directory) + if with_home: + os.environ['HOME'] = str(directory) + yield + finally: + os.chdir(cwd) + if home_exists: + os.environ['HOME'] = home + else: + del os.environ['HOME'] + return diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index a00a5764..c9c4a103 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1,10 +1,13 @@ +import os +import pathlib +import tempfile import unittest from datetime import date, datetime import pytest from sanic_openapi import doc -from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter +from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir from wrolpi.dates import set_timezone, now from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone from wrolpi.schema import validate_data @@ -315,3 +318,31 @@ class TestCommon(unittest.TestCase): def test_api_param_limiter(i, expected): limiter = api_param_limiter(100) # should never return an integer greater than 100. assert limiter(i) == expected + + +def test_chdir(): + """ + The current working directory can be changed temporarily using the `chdir` context manager. + """ + original = os.getcwd() + home = os.environ.get('HOME') + assert home + + with chdir(): + assert os.getcwd() != original + assert str(os.getcwd()).startswith('/tmp') + assert os.environ['HOME'] != os.getcwd() + # Replace $HOME + with chdir(with_home=True): + assert os.getcwd() != original + assert str(os.getcwd()).startswith('/tmp') + assert os.environ['HOME'] == os.getcwd() + + with tempfile.TemporaryDirectory() as d: + # Without replacing $HOME + with chdir(pathlib.Path(d), with_home=True): + assert os.getcwd() == d + assert os.environ['HOME'] == os.getcwd() + with chdir(pathlib.Path(d)): + assert os.getcwd() == d + assert os.environ['HOME'] != os.getcwd()