mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
commit
746a4452fb
10 changed files with 161 additions and 29 deletions
16
50x.html
Normal file
16
50x.html
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
<html>
|
||||
<body>
|
||||
|
||||
<h1>Error!</h1>
|
||||
|
||||
<p>nginx could not find WROLPi! Are your services running?</p>
|
||||
|
||||
<p><b>sudo systemctl status wrolpi-app</b></p>
|
||||
<p><b>sudo systemctl status wrolpi-api</b></p>
|
||||
|
||||
<p>If either of those services are not running, start it:</p>
|
||||
|
||||
<p><b>sudo systemctl start wrolpi-app</b></p>
|
||||
<p><b>sudo systemctl start wrolpi-api</b></p>
|
||||
</body>
|
||||
</html>
|
||||
11
install.sh
11
install.sh
|
|
@ -56,7 +56,7 @@ npm install -g 'git+https://github.com/pirate/readability-extractor'
|
|||
# Get the latest WROLPi code
|
||||
git --version
|
||||
git clone https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi || :
|
||||
(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git pull origin "${BRANCH}" --rebase) || exit 3
|
||||
(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 3
|
||||
|
||||
# Setup the virtual environment that main.py expects
|
||||
pip3 --version || (
|
||||
|
|
@ -89,12 +89,17 @@ sudo -u postgres psql -c '\l' | grep wrolpi || (
|
|||
|
||||
# Install the WROLPi nginx config over the default nginx config.
|
||||
cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf
|
||||
cp /opt/wrolpi/50x.html /var/www/50x.html
|
||||
/usr/sbin/nginx -s reload
|
||||
|
||||
# Create the WROLPi user
|
||||
grep wrolpi /etc/passwd || useradd -d /opt/wrolpi wrolpi -s "$(command -v bash)"
|
||||
grep wrolpi /etc/passwd || useradd -d /home/wrolpi wrolpi -s "$(command -v bash)"
|
||||
chown -R wrolpi:wrolpi /opt/wrolpi
|
||||
|
||||
# Create the media directory. This should be mounted by the maintainer.
|
||||
[ -d /media/wrolpi ] || mkdir /media/wrolpi
|
||||
chown -R wrolpi:wrolpi /media/wrolpi
|
||||
|
||||
# Install the systemd services
|
||||
cp /opt/wrolpi/systemd/wrolpi-api.service /etc/systemd/system/
|
||||
cp /opt/wrolpi/systemd/wrolpi-app.service /etc/systemd/system/
|
||||
|
|
@ -121,6 +126,8 @@ echo "
|
|||
|
||||
WROLPi has successfully been installed!
|
||||
|
||||
Mount your external hard drive to /media/wrolpi if you have one.
|
||||
|
||||
Start the WROLPi services using:
|
||||
|
||||
# sudo systemctl start wrolpi.target
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from selenium import webdriver
|
|||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.archive.models import Domain, Archive
|
||||
from wrolpi.common import get_media_directory, logger, chunks, extract_domain
|
||||
from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir
|
||||
from wrolpi.dates import now, strptime_ms, strftime_ms
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
|
||||
from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
|
||||
|
|
@ -26,7 +26,10 @@ ARCHIVE_SERVICE = 'http://archive:8080'
|
|||
|
||||
|
||||
def get_archive_directory() -> pathlib.Path:
|
||||
return get_media_directory() / 'archive'
|
||||
archive_directory = get_media_directory() / 'archive'
|
||||
if not archive_directory.is_dir():
|
||||
archive_directory.mkdir()
|
||||
return archive_directory
|
||||
|
||||
|
||||
def get_domain_directory(url: str) -> pathlib.Path:
|
||||
|
|
@ -141,6 +144,7 @@ def local_screenshot(url: str) -> bytes:
|
|||
# Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of the page.
|
||||
options = webdriver.ChromeOptions()
|
||||
options.add_argument('headless')
|
||||
options.add_argument('disable-gpu')
|
||||
options.add_argument('window-size=1280x720')
|
||||
|
||||
driver = webdriver.Chrome(chrome_options=options)
|
||||
|
|
@ -174,13 +178,16 @@ def local_archive(url: str):
|
|||
"""
|
||||
Perform an archive of the provided URL using local resources (without the Archive docker container).
|
||||
"""
|
||||
singlefile = local_singlefile(url)
|
||||
with tempfile.NamedTemporaryFile('wb') as fh:
|
||||
fh.write(singlefile)
|
||||
readability = local_extract_readability(fh.name, url)
|
||||
screenshot = local_screenshot(url)
|
||||
singlefile = singlefile.decode()
|
||||
return singlefile, readability, screenshot
|
||||
# Archives must be performed in the wrolpi home directory because chrome saves the screenshot there, and will
|
||||
# raise an error if global $HOME is not a real user directory. :(
|
||||
with chdir('/home/wrolpi', with_home=True):
|
||||
singlefile = local_singlefile(url)
|
||||
with tempfile.NamedTemporaryFile('wb') as fh:
|
||||
fh.write(singlefile)
|
||||
readability = local_extract_readability(fh.name, url)
|
||||
screenshot = local_screenshot(url)
|
||||
singlefile = singlefile.decode()
|
||||
return singlefile, readability, screenshot
|
||||
|
||||
|
||||
def do_archive(url: str) -> Archive:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import pytest
|
|||
|
||||
from modules.videos.api import refresh_queue
|
||||
from modules.videos.channel.lib import spread_channel_downloads, delete_channel
|
||||
from modules.videos.common import get_no_channel_directory
|
||||
from modules.videos.lib import upsert_video
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.test.common import create_channel_structure
|
||||
|
|
@ -139,9 +140,7 @@ class TestVideoAPI(TestAPI):
|
|||
session.commit()
|
||||
|
||||
# The special NO CHANNEL directory.
|
||||
from modules.videos.downloader import get_no_channel_directory
|
||||
no_channel_path = get_no_channel_directory()
|
||||
no_channel_path.mkdir()
|
||||
vid3 = no_channel_path / 'some video.mp4'
|
||||
vid3.touch()
|
||||
no_channel_subdir = no_channel_path / 'subdir'
|
||||
|
|
|
|||
|
|
@ -56,6 +56,26 @@ class ConfigError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
def get_videos_directory() -> pathlib.Path:
|
||||
"""
|
||||
Get the "videos" directory in the media directory. Make it if it does not exist.
|
||||
"""
|
||||
directory = get_media_directory() / 'videos'
|
||||
if not directory.is_dir():
|
||||
directory.mkdir(parents=True)
|
||||
return directory
|
||||
|
||||
|
||||
def get_no_channel_directory() -> pathlib.Path:
|
||||
"""
|
||||
Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist.
|
||||
"""
|
||||
directory = get_videos_directory() / 'NO CHANNEL'
|
||||
if not directory.is_dir():
|
||||
directory.mkdir(parents=True)
|
||||
return directory
|
||||
|
||||
|
||||
@before_startup
|
||||
def import_videos_config():
|
||||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
|
|
@ -291,7 +311,7 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
|
|||
path = str(path)
|
||||
|
||||
ignored_directories = {
|
||||
str(get_media_directory() / 'videos/NO CHANNEL'),
|
||||
str(get_no_channel_directory()),
|
||||
}
|
||||
|
||||
if os.path.isdir(path):
|
||||
|
|
|
|||
|
|
@ -11,16 +11,16 @@ from yt_dlp import YoutubeDL
|
|||
from yt_dlp.extractor import YoutubeTabIE
|
||||
from yt_dlp.utils import UnsupportedError, DownloadError
|
||||
|
||||
from wrolpi.common import logger, sanitize_link, extract_domain, \
|
||||
get_media_directory
|
||||
from wrolpi.common import logger, sanitize_link, extract_domain
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import Downloader, Download
|
||||
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import load_downloader_config, apply_info_json, get_channel_source_id
|
||||
from .lib import upsert_video, _refresh_videos, refresh_channel_videos
|
||||
from .common import load_downloader_config, apply_info_json, get_channel_source_id, get_no_channel_directory, \
|
||||
get_videos_directory
|
||||
from .lib import upsert_video, refresh_channel_videos
|
||||
from .models import Video, Channel
|
||||
from .video_url_resolver import video_url_resolver
|
||||
|
||||
|
|
@ -40,10 +40,6 @@ PREFERRED_VIDEO_EXTENSION = 'mp4'
|
|||
PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p'
|
||||
|
||||
|
||||
def get_no_channel_directory():
|
||||
return get_media_directory() / 'videos/NO CHANNEL'
|
||||
|
||||
|
||||
class ChannelDownloader(Downloader, ABC):
|
||||
"""
|
||||
Handling downloading of all Videos in a Channel.
|
||||
|
|
@ -94,6 +90,13 @@ class ChannelDownloader(Downloader, ABC):
|
|||
return True
|
||||
|
||||
|
||||
YT_DLP_BIN = pathlib.Path('/usr/local/bin/yt-dlp') # Location in docker container
|
||||
if not YT_DLP_BIN.is_file():
|
||||
YT_DLP_BIN = pathlib.Path('/opt/wrolpi/venv/bin/yt-dlp') # Use virtual environment location
|
||||
if not YT_DLP_BIN.is_file():
|
||||
logger.error('COULD NOT FIND YT-DLP!!!')
|
||||
|
||||
|
||||
class VideoDownloader(Downloader, ABC):
|
||||
"""
|
||||
Download a single video. Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`.
|
||||
|
|
@ -150,14 +153,14 @@ class VideoDownloader(Downloader, ABC):
|
|||
out_dir = get_no_channel_directory()
|
||||
if channel:
|
||||
out_dir = channel.directory.path
|
||||
out_dir.mkdir(exist_ok=True)
|
||||
out_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
try:
|
||||
video_path, entry = self.prepare_filename(url, out_dir)
|
||||
# Do the real download.
|
||||
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
|
||||
cmd = [
|
||||
'yt-dlp',
|
||||
str(YT_DLP_BIN),
|
||||
'-cw', # Continue downloads, do not clobber existing files.
|
||||
'-f', PREFERRED_VIDEO_FORMAT,
|
||||
'--write-subs',
|
||||
|
|
@ -254,7 +257,10 @@ def get_or_create_channel(source_id: str = None, link: str = None, url: str = No
|
|||
except UnknownChannel:
|
||||
pass
|
||||
|
||||
channel_directory = get_media_directory() / f'videos/{name}'
|
||||
# Channel does not exist. Create one in the video directory.
|
||||
channel_directory = get_videos_directory() / name
|
||||
if not channel_directory.is_dir():
|
||||
channel_directory.mkdir(parents=True)
|
||||
data = dict(
|
||||
name=name,
|
||||
url=url,
|
||||
|
|
|
|||
|
|
@ -439,7 +439,8 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
|
|||
if channel and not str(video_path).startswith(str(channel.directory.path)):
|
||||
raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}')
|
||||
|
||||
video.channel = channel
|
||||
if channel:
|
||||
video.channel_id = channel.id
|
||||
video.idempotency = idempotency
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -12,9 +12,14 @@ http {
|
|||
charset utf-8;
|
||||
include mime.types;
|
||||
|
||||
error_page 502 /error/50x.html;
|
||||
error_page 503 /error/50x.html;
|
||||
error_page 504 /error/50x.html;
|
||||
|
||||
server {
|
||||
location /favicon.ico {
|
||||
alias /opt/wrolpi/icon.ico;
|
||||
location ^~ /error/ {
|
||||
internal;
|
||||
alias /var/www/;
|
||||
}
|
||||
|
||||
location /apple-touch-icon.png {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import contextlib
|
|||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import queue
|
||||
import re
|
||||
|
|
@ -526,3 +527,42 @@ def partition(pred, iterable):
|
|||
"""
|
||||
t1, t2 = tee(iterable)
|
||||
return list(filter(pred, t2)), list(filterfalse(pred, t1))
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def chdir(directory: Union[pathlib.Path, str, None] = None, with_home: bool = False):
|
||||
"""
|
||||
Change the Python working directory in this context. If no directory is passed, a TemporaryDirectory will be used.
|
||||
|
||||
This also changes the $HOME environment variable to match.
|
||||
"""
|
||||
home_exists = 'HOME' in os.environ
|
||||
home = os.environ.get('HOME')
|
||||
cwd = os.getcwd()
|
||||
if directory is None:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
try:
|
||||
os.chdir(d)
|
||||
if with_home:
|
||||
os.environ['HOME'] = d
|
||||
yield
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
if home_exists:
|
||||
os.environ['HOME'] = home
|
||||
else:
|
||||
del os.environ['HOME']
|
||||
return
|
||||
|
||||
try:
|
||||
os.chdir(directory)
|
||||
if with_home:
|
||||
os.environ['HOME'] = str(directory)
|
||||
yield
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
if home_exists:
|
||||
os.environ['HOME'] = home
|
||||
else:
|
||||
del os.environ['HOME']
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import date, datetime
|
||||
|
||||
import pytest
|
||||
from sanic_openapi import doc
|
||||
|
||||
from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter
|
||||
from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir
|
||||
from wrolpi.dates import set_timezone, now
|
||||
from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone
|
||||
from wrolpi.schema import validate_data
|
||||
|
|
@ -315,3 +318,31 @@ class TestCommon(unittest.TestCase):
|
|||
def test_api_param_limiter(i, expected):
|
||||
limiter = api_param_limiter(100) # should never return an integer greater than 100.
|
||||
assert limiter(i) == expected
|
||||
|
||||
|
||||
def test_chdir():
|
||||
"""
|
||||
The current working directory can be changed temporarily using the `chdir` context manager.
|
||||
"""
|
||||
original = os.getcwd()
|
||||
home = os.environ.get('HOME')
|
||||
assert home
|
||||
|
||||
with chdir():
|
||||
assert os.getcwd() != original
|
||||
assert str(os.getcwd()).startswith('/tmp')
|
||||
assert os.environ['HOME'] != os.getcwd()
|
||||
# Replace $HOME
|
||||
with chdir(with_home=True):
|
||||
assert os.getcwd() != original
|
||||
assert str(os.getcwd()).startswith('/tmp')
|
||||
assert os.environ['HOME'] == os.getcwd()
|
||||
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
# Without replacing $HOME
|
||||
with chdir(pathlib.Path(d), with_home=True):
|
||||
assert os.getcwd() == d
|
||||
assert os.environ['HOME'] == os.getcwd()
|
||||
with chdir(pathlib.Path(d)):
|
||||
assert os.getcwd() == d
|
||||
assert os.environ['HOME'] != os.getcwd()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue