Merge pull request #76 from lrnselfreliance/release

Release
This commit is contained in:
lrnselfreliance 2022-01-16 13:40:33 -07:00 • committed by GitHub
commit 746a4452fb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 161 additions and 29 deletions

16
50x.html Normal file
View file

@ -0,0 +1,16 @@
<html>
<body>
<h1>Error!</h1>
<p>nginx could not find WROLPi! Are your services running?</p>
<p><b>sudo systemctl status wrolpi-app</b></p>
<p><b>sudo systemctl status wrolpi-api</b></p>
<p>If either of those services are not running, start it:</p>
<p><b>sudo systemctl start wrolpi-app</b></p>
<p><b>sudo systemctl start wrolpi-api</b></p>
</body>
</html>

View file

@ -56,7 +56,7 @@ npm install -g 'git+https://github.com/pirate/readability-extractor'
# Get the latest WROLPi code
git --version
git clone https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi || :
(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git pull origin "${BRANCH}" --rebase) || exit 3
(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 3
# Setup the virtual environment that main.py expects
pip3 --version || (
@ -89,12 +89,17 @@ sudo -u postgres psql -c '\l' | grep wrolpi || (
# Install the WROLPi nginx config over the default nginx config.
cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf
cp /opt/wrolpi/50x.html /var/www/50x.html
/usr/sbin/nginx -s reload
# Create the WROLPi user
grep wrolpi /etc/passwd || useradd -d /opt/wrolpi wrolpi -s "$(command -v bash)"
grep wrolpi /etc/passwd || useradd -d /home/wrolpi wrolpi -s "$(command -v bash)"
chown -R wrolpi:wrolpi /opt/wrolpi
# Create the media directory. This should be mounted by the maintainer.
[ -d /media/wrolpi ] || mkdir /media/wrolpi
chown -R wrolpi:wrolpi /media/wrolpi
# Install the systemd services
cp /opt/wrolpi/systemd/wrolpi-api.service /etc/systemd/system/
cp /opt/wrolpi/systemd/wrolpi-app.service /etc/systemd/system/
@ -121,6 +126,8 @@ echo "
WROLPi has successfully been installed!
Mount your external hard drive to /media/wrolpi if you have one.
Start the WROLPi services using:
# sudo systemctl start wrolpi.target

View file

@ -14,7 +14,7 @@ from selenium import webdriver
from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from wrolpi.common import get_media_directory, logger, chunks, extract_domain
from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir
from wrolpi.dates import now, strptime_ms, strftime_ms
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
@ -26,7 +26,10 @@ ARCHIVE_SERVICE = 'http://archive:8080'
def get_archive_directory() -> pathlib.Path:
return get_media_directory() / 'archive'
archive_directory = get_media_directory() / 'archive'
if not archive_directory.is_dir():
archive_directory.mkdir()
return archive_directory
def get_domain_directory(url: str) -> pathlib.Path:
@ -141,6 +144,7 @@ def local_screenshot(url: str) -> bytes:
# Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page.
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument('disable-gpu')
options.add_argument('window-size=1280x720')
driver = webdriver.Chrome(chrome_options=options)
@ -174,13 +178,16 @@ def local_archive(url: str):
"""
Perform an archive of the provided URL using local resources (without the Archive docker container).
"""
singlefile = local_singlefile(url)
with tempfile.NamedTemporaryFile('wb') as fh:
fh.write(singlefile)
readability = local_extract_readability(fh.name, url)
screenshot = local_screenshot(url)
singlefile = singlefile.decode()
return singlefile, readability, screenshot
# Archives must be performed in the wrolpi home directory because chrome saves the screenshot there, and will
# raise an error if global $HOME is not a real user directory. :(
with chdir('/home/wrolpi', with_home=True):
singlefile = local_singlefile(url)
with tempfile.NamedTemporaryFile('wb') as fh:
fh.write(singlefile)
readability = local_extract_readability(fh.name, url)
screenshot = local_screenshot(url)
singlefile = singlefile.decode()
return singlefile, readability, screenshot
def do_archive(url: str) -> Archive:

View file

@ -11,6 +11,7 @@ import pytest
from modules.videos.api import refresh_queue
from modules.videos.channel.lib import spread_channel_downloads, delete_channel
from modules.videos.common import get_no_channel_directory
from modules.videos.lib import upsert_video
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
@ -139,9 +140,7 @@ class TestVideoAPI(TestAPI):
session.commit()
# The special NO CHANNEL directory.
from modules.videos.downloader import get_no_channel_directory
no_channel_path = get_no_channel_directory()
no_channel_path.mkdir()
vid3 = no_channel_path / 'some video.mp4'
vid3.touch()
no_channel_subdir = no_channel_path / 'subdir'

View file

@ -56,6 +56,26 @@ class ConfigError(Exception):
pass
def get_videos_directory() -> pathlib.Path:
"""
Get the "videos" directory in the media directory. Make it if it does not exist.
"""
directory = get_media_directory() / 'videos'
if not directory.is_dir():
directory.mkdir(parents=True)
return directory
def get_no_channel_directory() -> pathlib.Path:
"""
Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist.
"""
directory = get_videos_directory() / 'NO CHANNEL'
if not directory.is_dir():
directory.mkdir(parents=True)
return directory
@before_startup
def import_videos_config():
"""Import channel settings to the DB. Existing channels will be updated."""
@ -291,7 +311,7 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
path = str(path)
ignored_directories = {
str(get_media_directory() / 'videos/NO CHANNEL'),
str(get_no_channel_directory()),
}
if os.path.isdir(path):

View file

@ -11,16 +11,16 @@ from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE
from yt_dlp.utils import UnsupportedError, DownloadError
from wrolpi.common import logger, sanitize_link, extract_domain, \
get_media_directory
from wrolpi.common import logger, sanitize_link, extract_domain
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import Downloader, Download
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import load_downloader_config, apply_info_json, get_channel_source_id
from .lib import upsert_video, _refresh_videos, refresh_channel_videos
from .common import load_downloader_config, apply_info_json, get_channel_source_id, get_no_channel_directory, \
get_videos_directory
from .lib import upsert_video, refresh_channel_videos
from .models import Video, Channel
from .video_url_resolver import video_url_resolver
@ -40,10 +40,6 @@ PREFERRED_VIDEO_EXTENSION = 'mp4'
PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p'
def get_no_channel_directory():
return get_media_directory() / 'videos/NO CHANNEL'
class ChannelDownloader(Downloader, ABC):
"""
Handling downloading of all Videos in a Channel.
@ -94,6 +90,13 @@ class ChannelDownloader(Downloader, ABC):
return True
YT_DLP_BIN = pathlib.Path('/usr/local/bin/yt-dlp') # Location in docker container
if not YT_DLP_BIN.is_file():
YT_DLP_BIN = pathlib.Path('/opt/wrolpi/venv/bin/yt-dlp') # Use virtual environment location
if not YT_DLP_BIN.is_file():
logger.error('COULD NOT FIND YT-DLP!!!')
class VideoDownloader(Downloader, ABC):
"""
Download a single video. Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`.
@ -150,14 +153,14 @@ class VideoDownloader(Downloader, ABC):
out_dir = get_no_channel_directory()
if channel:
out_dir = channel.directory.path
out_dir.mkdir(exist_ok=True)
out_dir.mkdir(exist_ok=True, parents=True)
try:
video_path, entry = self.prepare_filename(url, out_dir)
# Do the real download.
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
cmd = [
'yt-dlp',
str(YT_DLP_BIN),
'-cw', # Continue downloads, do not clobber existing files.
'-f', PREFERRED_VIDEO_FORMAT,
'--write-subs',
@ -254,7 +257,10 @@ def get_or_create_channel(source_id: str = None, link: str = None, url: str = No
except UnknownChannel:
pass
channel_directory = get_media_directory() / f'videos/{name}'
# Channel does not exist. Create one in the video directory.
channel_directory = get_videos_directory() / name
if not channel_directory.is_dir():
channel_directory.mkdir(parents=True)
data = dict(
name=name,
url=url,

View file

@ -439,7 +439,8 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
if channel and not str(video_path).startswith(str(channel.directory.path)):
raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}')
video.channel = channel
if channel:
video.channel_id = channel.id
video.idempotency = idempotency
try:

View file

@ -12,9 +12,14 @@ http {
charset utf-8;
include mime.types;
error_page 502 /error/50x.html;
error_page 503 /error/50x.html;
error_page 504 /error/50x.html;
server {
location /favicon.ico {
alias /opt/wrolpi/icon.ico;
location ^~ /error/ {
internal;
alias /var/www/;
}
location /apple-touch-icon.png {

View file

@ -4,6 +4,7 @@ import contextlib
import inspect
import json
import logging
import os
import pathlib
import queue
import re
@ -526,3 +527,42 @@ def partition(pred, iterable):
"""
t1, t2 = tee(iterable)
return list(filter(pred, t2)), list(filterfalse(pred, t1))
@contextlib.contextmanager
def chdir(directory: Union[pathlib.Path, str, None] = None, with_home: bool = False):
"""
Change the Python working directory in this context. If no directory is passed, a TemporaryDirectory will be used.
This also changes the $HOME environment variable to match.
"""
home_exists = 'HOME' in os.environ
home = os.environ.get('HOME')
cwd = os.getcwd()
if directory is None:
with tempfile.TemporaryDirectory() as d:
try:
os.chdir(d)
if with_home:
os.environ['HOME'] = d
yield
finally:
os.chdir(cwd)
if home_exists:
os.environ['HOME'] = home
else:
del os.environ['HOME']
return
try:
os.chdir(directory)
if with_home:
os.environ['HOME'] = str(directory)
yield
finally:
os.chdir(cwd)
if home_exists:
os.environ['HOME'] = home
else:
del os.environ['HOME']
return

View file

@ -1,10 +1,13 @@
import os
import pathlib
import tempfile
import unittest
from datetime import date, datetime
import pytest
from sanic_openapi import doc
from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter
from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir
from wrolpi.dates import set_timezone, now
from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone
from wrolpi.schema import validate_data
@ -315,3 +318,31 @@ class TestCommon(unittest.TestCase):
def test_api_param_limiter(i, expected):
limiter = api_param_limiter(100) # should never return an integer greater than 100.
assert limiter(i) == expected
def test_chdir():
"""
The current working directory can be changed temporarily using the `chdir` context manager.
"""
original = os.getcwd()
home = os.environ.get('HOME')
assert home
with chdir():
assert os.getcwd() != original
assert str(os.getcwd()).startswith('/tmp')
assert os.environ['HOME'] != os.getcwd()
# Replace $HOME
with chdir(with_home=True):
assert os.getcwd() != original
assert str(os.getcwd()).startswith('/tmp')
assert os.environ['HOME'] == os.getcwd()
with tempfile.TemporaryDirectory() as d:
# Without replacing $HOME
with chdir(pathlib.Path(d), with_home=True):
assert os.getcwd() == d
assert os.environ['HOME'] == os.getcwd()
with chdir(pathlib.Path(d)):
assert os.getcwd() == d
assert os.environ['HOME'] != os.getcwd()