@@ -110,9 +114,10 @@ export function VideoCard({video}) {
-
- {channel.name}
-
+ {channel &&
+
+ {channel.name}
+ }
{upload_date}
@@ -229,6 +234,27 @@ export function humanFileSize(bytes, si = false, dp = 1) {
return bytes.toFixed(dp) + ' ' + units[u];
}
+export function humanNumber(num, dp = 1) {
+ // Convert large numbers to a more human readable format.
+ // >> humanNumber(1000)
+ // 1.0k
+ // >> humanNumber(1500000)
+ // 1.5m
+ const divisor = 1000;
+ if (Math.abs(num) < divisor) {
+ return num;
+ }
+ const units = ['k', 'm', 'b'];
+ let i = -1;
+ const r = 10 ** dp;
+ do {
+ num /= divisor;
+ ++i;
+ } while (Math.round(Math.abs(num) * r) / r >= divisor && i < units.length - 1);
+
+ return num.toFixed(dp) + units[i];
+}
+
export class APIForm extends React.Component {
constructor(props) {
@@ -317,7 +343,7 @@ export class APIForm extends React.Component {
this.setState({
disabled: true,
error: false,
- errors: this.getEmptyErrors,
+ errors: this.getEmptyErrors(),
loading: true,
success: false,
})
diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js
new file mode 100644
index 00000000..de4356cd
--- /dev/null
+++ b/app/src/components/Upload.js
@@ -0,0 +1,74 @@
+import React from "react";
+import {Button, Form, Header, Segment, TextArea} from "semantic-ui-react";
+import {postDownload} from "../api";
+
+const validUrl = /^(http|https):\/\/[^ "]+$/;
+
+class Downloader extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ urls: '',
+ valid: true,
+ };
+ }
+
+ submitDownload = async () => {
+ if (this.state.urls) {
+ let response = await postDownload(this.state.urls);
+ if (response.status === 204) {
+ this.setState({urls: ''});
+ }
+ }
+ }
+
+ handleInputChange = async (event, {name, value}) => {
+ this.setState({[name]: value}, this.validateUrls);
+ }
+
+ validateUrls = async () => {
+ // Validate that all URLs in state are valid.
+ if (this.state.urls) {
+ let urls = this.state.urls.split(/\r?\n/);
+ let valid = true;
+ urls.forEach((url) => {
+ if (valid && url && !validUrl.test(url)) {
+ valid = false;
+ }
+ })
+ this.setState({valid});
+ }
+ }
+
+ render() {
+ let textareaPlaceholder = 'Enter one URL per line';
+ let disabled = !this.state.urls || !this.state.valid;
+
+ return (
+
+ )
+ }
+}
+
+export class Saver extends React.Component {
+
+ render() {
+ return (
+ <>
+
+
+
+ >
+ )
+ }
+}
diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js
index 152aebba..deb0e27c 100644
--- a/app/src/components/VideoPlayer.js
+++ b/app/src/components/VideoPlayer.js
@@ -3,7 +3,7 @@ import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {deleteVideo, favoriteVideo} from "../api";
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
import {Link} from "react-router-dom";
-import {humanFileSize, uploadDate, VideoCard} from "./Common";
+import {humanFileSize, humanNumber, uploadDate, VideoCard} from "./Common";
import {Confirm, Container, Segment, Tab} from "semantic-ui-react";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
@@ -24,15 +24,15 @@ function VideoPage(props) {
props.history.goBack();
}
- let videoUrl = `${MEDIA_PATH}/${channel.directory}/${encodeURIComponent(video.video_path)}`;
+ let videoUrl = `${MEDIA_PATH}/${encodeURIComponent(video.video_path)}`;
let posterUrl = null;
if (video.poster_path) {
- posterUrl = `${MEDIA_PATH}/${channel.directory}/${encodeURIComponent(video.poster_path)}`;
+ posterUrl = `${MEDIA_PATH}/${encodeURIComponent(video.poster_path)}`;
}
let captionsUrl = null;
if (video.caption_path) {
- captionsUrl = `${MEDIA_PATH}/${channel.directory}/${encodeURIComponent(video.caption_path)}`;
+ captionsUrl = `${MEDIA_PATH}/${encodeURIComponent(video.caption_path)}`;
}
let description = 'No description available.';
@@ -78,16 +78,19 @@ function VideoPage(props) {
let statisticsPane = {
menuItem: 'Statistics', render: () =>
Size
- {humanFileSize(video.size)}
+ {video.size ? humanFileSize(video.size) : '???'}
+
+ Source URL
+ {video.url || 'N/A'}
Source ID
- {video.source_id}
+ {video.source_id || 'N/A'}
View Count
- {viewCount}
+ {viewCount ? humanNumber(viewCount) : 'N/A'}
File Name
- {video.channel.directory}/{video.video_path}
+ {video.video_path}
}
@@ -117,9 +120,9 @@ function VideoPage(props) {
{video.title}
{video.upload_date && {uploadDate(video.upload_date)} }
-
+ {channel &&
{channel.name}
-
+ }
diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js
index d539957f..282184ae 100644
--- a/app/src/components/Videos.js
+++ b/app/src/components/Videos.js
@@ -10,12 +10,11 @@ import Paginator, {
scrollToTop,
searchOrders,
secondsToString,
- uploadDate,
VideoCards,
videoOrders
} from "./Common"
import VideoPage from "./VideoPlayer";
-import {download, downloadHistory, getChannel, getStatistics, getVideo, refresh, searchVideos} from "../api";
+import {download, getChannel, getStatistics, getVideo, refresh, searchVideos} from "../api";
import {Button, Dropdown, Form, Grid, Header, Icon, Input, Loader, Segment, Statistic, Tab} from "semantic-ui-react";
import * as QueryString from 'query-string';
import Container from "semantic-ui-react/dist/commonjs/elements/Container";
@@ -28,15 +27,10 @@ class ManageVideos extends React.Component {
super(props);
this.state = {
streamUrl: null,
- downloadHistory: null,
mostRecentDownload: '',
}
}
- async componentDidMount() {
- await this.downloadHistory();
- }
-
download = async (e) => {
e.preventDefault();
let response = await download();
@@ -53,15 +47,6 @@ class ManageVideos extends React.Component {
}
}
- async downloadHistory() {
- let response = await downloadHistory();
- if (response && response.history.length > 0) {
- let history = response.history;
- let mostRecentDownload = uploadDate(history[0]['info_date']);
- this.setState({downloadHistory: history, mostRecentDownload: mostRecentDownload});
- }
- }
-
render() {
return (
@@ -75,7 +60,6 @@ class ManageVideos extends React.Component {
Download any missing videos
- Most recent download: {this.state.mostRecentDownload}
@@ -102,6 +86,7 @@ export class VideoWrapper extends React.Component {
prev: null,
next: null,
channel: null,
+ no_channel: null,
}
}
@@ -112,8 +97,13 @@ export class VideoWrapper extends React.Component {
async fetchVideo() {
// Get and display the Video specified in the Router match
let [video, prev, next] = await getVideo(this.props.match.params.video_id);
- let channel = await getChannel(this.props.match.params.channel_link);
- this.setState({video, prev, next, channel}, scrollToTop);
+ let channel_link = this.props.match.params.channel_link;
+ let channel = channel_link ? await getChannel(channel_link) : null;
+ let no_channel = false;
+ if (!channel) {
+ no_channel = true;
+ }
+ this.setState({video, prev, next, channel, no_channel}, scrollToTop);
}
async componentDidUpdate(prevProps, prevState) {
@@ -125,7 +115,7 @@ export class VideoWrapper extends React.Component {
}
render() {
- if (this.state.video && this.state.channel) {
+ if (this.state.video && (this.state.no_channel || this.state.channel)) {
return
} else {
return
diff --git a/conftest.py b/conftest.py
index d2e0ba61..bbcf7a76 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,6 @@
# Import conftest for test fixtures
-import os
-
+from wrolpi.common import import_modules
from wrolpi.conftest import * # noqa
-os.environ['DB_PORT'] = '54321'
+# Import modules and their conftest.py.
+import_modules()
diff --git a/docker-compose.yml b/docker-compose.yml
index 2472a21c..02df6191 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,7 +8,6 @@ services:
- POSTGRES_PASSWORD=wrolpi
ports:
- '54321:5432'
- - '5432:5432'
healthcheck:
test: [ 'CMD-SHELL', 'pg_isready -U postgres' ]
diff --git a/main.py b/main.py
index c6722b4e..73b70d6c 100755
--- a/main.py
+++ b/main.py
@@ -6,11 +6,13 @@ import logging
import sys
import pytz
+from sanic import Sanic
-from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS
-from wrolpi.common import logger, get_config
+from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, after_startup, limit_concurrent
+from wrolpi.common import logger, get_config, wrol_mode_enabled, import_modules
from wrolpi.dates import set_timezone
-from wrolpi.vars import PROJECT_DIR, MODULES_DIR
+from wrolpi.downloader import download_manager
+from wrolpi.vars import PROJECT_DIR
logger = logger.getChild('wrolpi-main')
@@ -40,7 +42,7 @@ def db_main(args):
return 0
-async def main():
+async def main(loop):
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='count')
@@ -70,18 +72,7 @@ async def main():
set_timezone(tz)
# Import the API in every module. Each API should attach itself to `root_api`.
- try:
- modules = [i.name for i in MODULES_DIR.iterdir() if i.is_dir() and not i.name.startswith('_')]
- for module in modules:
- module = f'modules.{module}.api'
- logger.debug(f'Importing {module}')
- __import__(module, globals(), locals(), [], 0)
- except ImportError as e:
- logger.fatal('No modules could be found!', exc_info=e)
- raise
-
- if not modules:
- raise Exception('No modules could be found!')
+ import_modules()
# Run the startup functions
for func in BEFORE_STARTUP_FUNCTIONS:
@@ -93,8 +84,8 @@ async def main():
except Exception as e:
logger.warning(f'Startup {func} failed!', exc_info=e)
- # Run the API
- return root_api.main(args)
+ # Run the API.
+ return root_api.main(loop, args)
async def set_log_level(args):
@@ -113,7 +104,32 @@ async def set_log_level(args):
logger.warning(f'Logging level: {level_name}')
+@after_startup
+@limit_concurrent(1)
+def periodic_downloads(app: Sanic, loop):
+ """
+ A simple function that perpetually calls downloader_manager.do_downloads() after sleeping.
+ """
+ if wrol_mode_enabled():
+ logger.warning(f'Not starting download worker because WROL Mode is enabled.')
+ return
+
+ # Set all downloads to new.
+ download_manager.reset_downloads()
+
+ async def _periodic_download():
+ download_manager.renew_recurring_downloads()
+ download_manager.delete_old_once_downloads()
+ await download_manager.do_downloads()
+ seconds = 60
+ logger.debug(f'Waiting {seconds} seconds to start next download')
+ await asyncio.sleep(seconds)
+ app.add_task(_periodic_download())
+
+ app.add_task(_periodic_download())
+
+
if __name__ == '__main__':
- loop = asyncio.get_event_loop()
- result = loop.run_until_complete(main())
+ loop_ = asyncio.get_event_loop()
+ result = loop_.run_until_complete(main(loop_))
sys.exit(result)
diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py
index 2d48cc88..eaaf2f00 100644
--- a/modules/archive/__init__.py
+++ b/modules/archive/__init__.py
@@ -1,6 +1,31 @@
+from abc import ABC
+
from wrolpi.common import logger
+from wrolpi.downloader import Downloader, Download
+from wrolpi.errors import UnrecoverableDownloadError
+from . import lib
from .api import bp # noqa
PRETTY_NAME = 'Archive'
logger = logger.getChild(__name__)
+
+
+class ArchiveDownloader(Downloader, ABC):
+ @staticmethod
+ def valid_url(url: str) -> bool:
+ """
+ Archiver will attempt to archive anything, so it should be last!
+ """
+ return True
+
+ def do_download(self, download: Download):
+ if download.attempts > 3:
+ raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
+
+ lib.new_archive(download.url, sync=True)
+ return True
+
+
+# Archive downloader is the last downloader which should be used.
+archive_downloader = ArchiveDownloader(priority=100)
diff --git a/modules/archive/api.py b/modules/archive/api.py
index bd6d7f86..e7b35a9f 100644
--- a/modules/archive/api.py
+++ b/modules/archive/api.py
@@ -64,7 +64,7 @@ async def search_archives(_: Request, data: dict):
return json_response(ret)
-@bp.post(':refresh')
+@bp.post('/refresh')
@wrol_mode_check
async def refresh_archives(_: Request):
asyncio.ensure_future(lib.refresh_archives())
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 326212e1..8c4bbc11 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -6,17 +6,17 @@ import pathlib
import re
from itertools import groupby
from typing import Iterator
-from urllib.parse import urlparse
import requests
from bs4 import BeautifulSoup
from sqlalchemy.orm import Session
from modules.archive.models import URL, Domain, Archive
-from wrolpi.common import get_media_directory, logger, chunks
+from wrolpi.common import get_media_directory, logger, chunks, extract_domain
from wrolpi.dates import now, strptime_ms, strftime_ms
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import InvalidDomain, UnknownURL, PendingArchive, InvalidArchive
+from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@@ -27,21 +27,6 @@ def get_archive_directory() -> pathlib.Path:
return get_media_directory() / 'archive'
-def extract_domain(url):
- """
- Extract the domain from a URL. Remove leading www.
-
- >>> extract_domain('https://www.example.com/foo')
- 'example.com'
- """
- parsed = urlparse(url)
- domain = parsed.netloc
- if domain.startswith('www.'):
- # Remove leading www.
- domain = domain[4:]
- return domain
-
-
def get_domain_directory(url: str) -> pathlib.Path:
"""
Get the archive directory for a particular domain.
@@ -161,7 +146,7 @@ def new_archive(url: str, sync: bool = False):
url_.latest_id = archive_id
url_.latest_datetime = now()
- if sync:
+ if PYTEST or sync:
return _do_archive(url, archive_id)
else:
# Run the real archive process in the future.
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index ff68f8c0..d0a779fb 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -312,7 +312,7 @@ class TestArchive(TestAPI):
archive: Archive = session.query(Archive).one()
self.assertEqual(url.latest, archive)
# latest_datetime is set, the timestamp in the file name is assumed to be UTC.
- self.assertEqual(str(url.latest_datetime), '2021-10-05 22:20:10.346823-06:00')
+ self.assertEqual(str(url.latest_datetime), '2021-10-05 16:20:10.346823-06:00')
# Running the refresh does not result in a new archive.
_refresh_archives()
diff --git a/modules/inventory/models.py b/modules/inventory/models.py
index 0b285bc6..64d4c365 100644
--- a/modules/inventory/models.py
+++ b/modules/inventory/models.py
@@ -27,8 +27,8 @@ class Item(Base, ModelHelper):
inventory = relationship('Inventory', primaryjoin="Item.inventory_id==Inventory.id", back_populates='items')
def __repr__(self):
- return f'- '
+ return f'
- '
class Inventory(Base, ModelHelper):
@@ -43,7 +43,7 @@ class Inventory(Base, ModelHelper):
items: InstrumentedList = relationship('Item', foreign_keys='Item.inventory_id')
def __repr__(self):
- return f'
'
+ return f''
def dict(self):
d = super().dict()
diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py
index e69de29b..8bfe14d6 100644
--- a/modules/videos/__init__.py
+++ b/modules/videos/__init__.py
@@ -0,0 +1 @@
+from modules.videos import downloader
diff --git a/modules/videos/api.py b/modules/videos/api.py
index 212c0f73..dd629c39 100644
--- a/modules/videos/api.py
+++ b/modules/videos/api.py
@@ -1,28 +1,25 @@
import asyncio
-import random
from functools import wraps
from http import HTTPStatus
-from multiprocessing import Event
-import requests
from sanic import Blueprint, response
from sanic.request import Request
from wrolpi import before_startup
from wrolpi.common import create_websocket_feed, get_sanic_url, \
- wrol_mode_check, ProgressReporter, get_config
+ wrol_mode_check
from wrolpi.common import logger
from wrolpi.root_api import add_blueprint, json_response
from wrolpi.schema import validate_doc, JSONErrorResponse
from .channel.api import channel_bp
-from .downloader import update_channels, download_all_missing_videos
-from .lib import process_video_meta_data, _refresh_videos, get_statistics, get_download_history
+from .channel.lib import download_channel, spread_channel_downloads
+from .lib import _refresh_videos, get_statistics
from .schema import StreamResponse, \
- FavoriteRequest, FavoriteResponse, VideosStatisticsResponse, DownloadHistoryResponse
+ FavoriteRequest, FavoriteResponse, VideosStatisticsResponse
from .video.api import video_bp
from .video.lib import set_video_favorite
-content_bp = Blueprint('Video Content', '/api/videos')
+content_bp = Blueprint('VideoContent', '/api/videos')
bp = Blueprint('Videos', '/api/videos').group(
content_bp, # view and manage video content and settings
channel_bp, # view and manage channels
@@ -35,8 +32,8 @@ logger = logger.getChild(__name__)
refresh_queue, refresh_event = create_websocket_feed('refresh', '/feeds/refresh', content_bp)
-@content_bp.post(':refresh')
-@content_bp.post(':refresh/')
+@content_bp.post('/refresh')
+@content_bp.post('/refresh/')
@validate_doc(
summary='Search for videos that have previously been downloaded and stored.',
produces=StreamResponse,
@@ -63,8 +60,7 @@ async def refresh(_, link: str = None):
await refresh_videos(channel_links)
refresh_logger.info('refresh complete')
- except Exception as e:
- refresh_queue.put({'error': 'Refresh failed. See server logs.', 'message': str(e)})
+ except Exception:
raise
finally:
refresh_event.clear()
@@ -78,8 +74,8 @@ async def refresh(_, link: str = None):
download_queue, download_event = create_websocket_feed('download', '/feeds/download', content_bp)
-@content_bp.post(':download')
-@content_bp.post(':download/')
+@content_bp.post('/download')
+@content_bp.post('/download/')
@validate_doc(
summary='Update channel catalogs, download any missing videos',
produces=StreamResponse,
@@ -88,7 +84,7 @@ download_queue, download_event = create_websocket_feed('download', '/feeds/downl
],
)
@wrol_mode_check
-async def download(_, link: str = None):
+def download(_, link: str = None):
download_logger = logger.getChild('download')
stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/download')
@@ -98,44 +94,18 @@ async def download(_, link: str = None):
if refresh_event.is_set():
return response.json({'error': 'Refresh is running. Cannot download.'})
- download_event.set()
+ download_channel(link)
- async def do_download():
- reporter = ProgressReporter(download_queue, 2)
- reporter.set_progress_total(0, 1)
- reporter.send_progress(0, 1, 'Download started')
-
- try:
- update_channels(reporter, link)
- download_logger.info('Updated all channel catalogs')
- download_all_missing_videos(reporter, link)
- reporter.finish(1, 'All videos have been downloaded')
-
- # Fill in any missing data for all videos.
- reporter.message(0, 'Processing and cleaning files')
- process_video_meta_data()
- reporter.finish(0, 'Processing and cleaning complete')
-
- download_logger.info('download complete')
- except Exception as e:
- logger.fatal(f'Download failed: {e}')
- reporter.error(0, 'Download failed. See server logs.')
- raise
- finally:
- download_event.clear()
-
- coro = do_download()
- asyncio.ensure_future(coro)
download_logger.debug('do_download scheduled')
return response.json({'code': 'stream-started', 'stream_url': stream_url})
@wraps(_refresh_videos)
async def refresh_videos(channel_links: list = None):
- return _refresh_videos(refresh_queue, channel_links=channel_links)
+ return _refresh_videos(channel_links=channel_links)
-@content_bp.post(':favorite')
+@content_bp.post('/favorite')
@validate_doc(
summary='Toggle the favorite flag on a video',
consumes=FavoriteRequest,
@@ -160,56 +130,6 @@ async def statistics(_: Request):
return json_response(ret, HTTPStatus.OK)
-@content_bp.get('/download_history')
-@validate_doc(
- summary='Retrieve video download history',
- produces=DownloadHistoryResponse,
-)
-async def download_history(_: Request):
- history = await get_download_history()
- ret = dict(history=history)
- return json_response(ret, HTTPStatus.OK)
-
-
-MIN_DOWNLOAD_FREQUENCY = 60 * 60 * 4 # 4 hours
-MAX_DOWNLOAD_FREQUENCY = 60 * 60 * 12 # 12 hours
-PERIODIC_DOWNLOAD_EVENT = Event()
-
-
-async def _periodic_download(min_download_frequency: int, max_download_frequency: int):
- """
- Wait some amount of time, download, then schedule the next download.
- """
- sleep_seconds = random.randint(min_download_frequency, max_download_frequency)
- logger.debug(f'Waiting {sleep_seconds} seconds before next periodic download')
- await asyncio.sleep(sleep_seconds)
-
- url = get_sanic_url(path='/api/videos:download')
- resp = requests.post(url)
- if resp.status_code != HTTPStatus.OK and resp.status_code != HTTPStatus.CONFLICT:
- # Download failed and wasn't already running.
- logger.warning(f'Periodic download failed with status_code={resp.status_code}')
- logger.warning(f'Periodic download response={resp}')
-
- # Schedule the next download
- asyncio.ensure_future(_periodic_download(min_download_frequency, max_download_frequency))
-
-
@before_startup
-def start_periodic_download():
- """
- Periodically download the videos.
- """
- if not PERIODIC_DOWNLOAD_EVENT.is_set():
- PERIODIC_DOWNLOAD_EVENT.set()
- logger.info(f'Starting periodic download')
-
- config = get_config()
- try:
- min_download_frequency = int(config.get('min_download_frequency', MIN_DOWNLOAD_FREQUENCY))
- max_download_frequency = int(config.get('max_download_frequency', MAX_DOWNLOAD_FREQUENCY))
- except Exception:
- logger.fatal(f'Failed to get min/max download frequencies from config!', exc_info=True)
- raise
-
- asyncio.ensure_future(_periodic_download(min_download_frequency, max_download_frequency))
+def startup_spread_channel_downloads():
+ spread_channel_downloads()
diff --git a/modules/videos/captions.py b/modules/videos/captions.py
index bd18d93a..f44d8235 100755
--- a/modules/videos/captions.py
+++ b/modules/videos/captions.py
@@ -1,4 +1,5 @@
#! /usr/bin/env python3
+import pathlib
from pathlib import Path
from typing import Generator, List, Union
@@ -7,8 +8,7 @@ import webvtt
from wrolpi.common import logger, chunks
from wrolpi.db import get_db_session
-from wrolpi.errors import UnknownCaptionFile
-from .common import get_absolute_video_caption
+from wrolpi.media_path import MediaPath
from .models import Video
@@ -43,9 +43,11 @@ def process_captions(video: Video):
"""
Parse and insert captions for a video record.
"""
- caption_path = get_absolute_video_caption(video)
- if not caption_path.exists():
- raise UnknownCaptionFile()
+ if isinstance(video.caption_path, MediaPath):
+ caption_path = video.caption_path.path
+ else:
+ caption_path = pathlib.Path(video.caption_path)
+
try:
lines = get_unique_caption_lines(str(caption_path))
block = '\n'.join(lines)
diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py
index bc0ce555..48eb5333 100644
--- a/modules/videos/channel/api.py
+++ b/modules/videos/channel/api.py
@@ -5,8 +5,8 @@ from sanic import response, Blueprint
from sanic.request import Request
from wrolpi.common import logger, wrol_mode_check, get_relative_to_media_directory, make_media_directory
-from wrolpi.root_api import json_response
from wrolpi.errors import UnknownDirectory
+from wrolpi.root_api import json_response
from wrolpi.schema import validate_doc, JSONErrorResponse
from .lib import get_minimal_channels, delete_channel, update_channel, get_channel, create_channel
from ..schema import ChannelsResponse, ChannelResponse, ChannelPostRequest, \
@@ -27,7 +27,7 @@ async def get_channels(_: Request):
return json_response({'channels': channels})
-@channel_bp.route('/', methods=['GET', 'OPTIONS'])
+@channel_bp.route('/', methods=['GET', 'OPTIONS'])
@validate_doc(
summary='Get a Channel',
produces=ChannelResponse,
@@ -36,7 +36,7 @@ async def get_channels(_: Request):
),
)
def channel_get(_: Request, link: str):
- channel = get_channel(link)
+ channel = get_channel(link=link)
channel.pop('info_json')
return json_response({'channel': channel})
@@ -61,6 +61,15 @@ def channel_post(_: Request, data: dict):
else:
raise
+ if download_frequency := data.get('download_frequency'):
+ try:
+ download_frequency = int(download_frequency)
+ except ValueError:
+ if download_frequency in ('null', 'None'):
+ download_frequency = None
+
+ data['download_frequency'] = download_frequency
+
channel = create_channel(data)
# Refresh the videos asynchronously
@@ -72,12 +81,11 @@ def channel_post(_: Request, data: dict):
{'Location': f'/api/videos/channels/{channel["link"]}'})
-@channel_bp.put('/')
-@channel_bp.patch('/')
+@channel_bp.put('/')
+@channel_bp.patch('/')
@validate_doc(
summary='Update a Channel',
consumes=ChannelPutRequest,
- produces=SuccessResponse,
responses=(
(HTTPStatus.NOT_FOUND, JSONErrorResponse),
(HTTPStatus.BAD_REQUEST, JSONErrorResponse),
@@ -90,7 +98,7 @@ def channel_update(_: Request, link: str, data: dict):
headers={'Location': f'/api/videos/channels/{channel.link}'})
-@channel_bp.delete('/')
+@channel_bp.delete('/')
@validate_doc(
summary='Delete a Channel',
produces=SuccessResponse,
diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index 51913e1d..d34d47d8 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -1,16 +1,19 @@
-from typing import List, Dict
+from collections import defaultdict
+from datetime import timedelta
+from typing import List, Dict, Union
from sqlalchemy.orm.exc import NoResultFound
-from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory
+from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory, logger
+from wrolpi.dates import today
from wrolpi.db import get_db_session, get_db_curs
+from wrolpi.downloader import download_manager, Download
from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError
from ..common import check_for_channel_conflicts
from ..lib import save_channels_config
from ..models import Channel
-DEFAULT_DOWNLOAD_FREQUENCY = 60 * 60 * 24 * 7 # weekly
-DEFAULT_DOWNLOAD_TIMEOUT = 60.0 * 10.0 # Ten minutes
+logger = logger.getChild(__name__)
async def get_minimal_channels() -> List[dict]:
@@ -21,7 +24,7 @@ async def get_minimal_channels() -> List[dict]:
# Get all channels, even if they don't have videos.
stmt = '''
SELECT
- c.id, name, link, directory, url, download_frequency, info_date, next_download
+ c.id, name, link, directory, url, download_frequency, info_date
FROM
channel AS c
ORDER BY LOWER(name)
@@ -64,15 +67,69 @@ async def get_channels_video_count() -> Dict[int, int]:
return video_counts
-@run_after(save_channels_config)
-def delete_channel(link):
- with get_db_session(commit=True) as session:
- try:
- channel = session.query(Channel).filter_by(link=link).one()
- except NoResultFound:
- raise UnknownChannel()
+def get_channel(source_id: str = None, link: str = None, url: str = None, return_dict: bool = True
+ ) -> Union[dict, Channel]:
+ """
+ Attempt to find a Channel using the provided params. The params are in order of reliability.
- channel.delete_with_videos()
+ Raises UnknownChannel if no channel is found.
+ """
+ with get_db_session() as session:
+ # Try by source_id first.
+ channel = None
+ if source_id:
+ channel = session.query(Channel).filter_by(source_id=source_id).one_or_none()
+ if not channel and link:
+ channel = session.query(Channel).filter_by(link=link).one_or_none()
+ if not channel and url:
+ channel = session.query(Channel).filter_by(url=url).one_or_none()
+
+ if not channel:
+ raise UnknownChannel(f'No channel matches {link=} {source_id=} {url=}')
+
+ logger.debug(f'Found {channel=} using {source_id=} {link=} {url=}')
+ return channel.dict() if return_dict else channel
+
+
+def _spread_by_frequency(channels: List[Channel]):
+ channels_by_frequency = defaultdict(lambda: [])
+ for channel in channels:
+ channels_by_frequency[channel.download_frequency].append(channel)
+
+ for frequency, channels in channels_by_frequency.items():
+ # The seconds between each download.
+ chunk = frequency // len(channels)
+ for channel in channels:
+ # My position in the channel group.
+ index = channels.index(channel)
+ # My next download will be distributed by my frequency and my position.
+ position = chunk * index
+ next_download = today() + timedelta(seconds=frequency + position)
+ yield dict(url=channel.url, frequency=frequency, next_download=next_download)
+
+
+def spread_channel_downloads():
+ """
+ Channels should be downloaded in a manner that is spread out over their frequency. For example, three channels
+ with a frequency of a week should be downloaded on different days that week.
+ """
+ with get_db_session(commit=True) as session:
+ channels = list(session.query(Channel).filter(
+ Channel.url != None,
+ Channel.url != '',
+ Channel.download_frequency != None
+ ).order_by(Channel.link).all()) # noqa
+
+ url_next_download = _spread_by_frequency(channels)
+
+ for info in url_next_download:
+ download = download_manager.get_download(session, url=info['url'])
+ if not download:
+ download = Download(url=info['url'])
+ session.add(download)
+
+ download.frequency = info['frequency']
+ download.next_download = info['next_download']
@run_after(save_channels_config)
@@ -101,7 +158,10 @@ def update_channel(data, link):
try:
data['download_frequency'] = int(data['download_frequency'])
except ValueError:
- raise APIError('Invalid download frequency')
+ if data['download_frequency'] in ('null', 'None', ''):
+ data['download_frequency'] = None
+ else:
+ raise APIError(f'Invalid download frequency {data["download_frequency"]}')
if data.get('match_regex') in ('None', 'null'):
data['match_regex'] = None
@@ -119,35 +179,28 @@ def update_channel(data, link):
# Apply the changes now that we've OK'd them
channel.update(data)
+ spread_channel_downloads()
+
return channel
-def get_channel(link) -> dict:
- """
- Get a Channel by it's `link`. Raise UnknownChannel if it does not exist.
- """
- with get_db_session() as session:
- try:
- channel = session.query(Channel).filter_by(link=link).one()
- except NoResultFound:
- raise UnknownChannel()
- return channel.dict()
-
-
@run_after(save_channels_config)
-def create_channel(data: dict) -> dict:
+@run_after(spread_channel_downloads)
+def create_channel(data: dict, return_dict: bool = True) -> Union[Channel, dict]:
"""
Create a new Channel. Check for conflicts with existing Channels.
"""
with get_db_session(commit=True) as session:
+ link = sanitize_link(data['name'])
try:
# Verify that the URL/Name/Link aren't taken
check_for_channel_conflicts(
session,
url=data.get('url'),
name=data['name'],
- link=sanitize_link(data['name']),
+ link=link,
directory=str(data['directory']),
+ source_id=data.get('source_id'),
)
except APIError as e:
raise ValidationError() from e
@@ -156,13 +209,34 @@ def create_channel(data: dict) -> dict:
name=data['name'],
url=data.get('url'),
match_regex=data.get('match_regex'),
- link=sanitize_link(data['name']),
+ link=link,
directory=str(data['directory']),
- download_frequency=data.get('download_frequency', DEFAULT_DOWNLOAD_FREQUENCY),
+ download_frequency=data.get('download_frequency'),
+ source_id=data.get('source_id'),
)
session.add(channel)
session.commit()
session.flush()
session.refresh(channel)
- return channel.dict()
+ if return_dict:
+ return channel.dict()
+ else:
+ return channel
+
+
+@run_after(save_channels_config)
+def delete_channel(link):
+ with get_db_session(commit=True) as session:
+ try:
+ channel = session.query(Channel).filter_by(link=link).one()
+ except NoResultFound:
+ raise UnknownChannel()
+
+ channel.delete_with_videos()
+
+
+def download_channel(link: str):
+ channel = get_channel(link=link, return_dict=False)
+ with get_db_session(commit=True) as session:
+ download_manager.create_download(channel.url, session)
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index aea45baf..656e2cc7 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -1,4 +1,5 @@
import json
+import os
import pathlib
import tempfile
from datetime import timedelta
@@ -9,14 +10,16 @@ import mock
import pytest
from modules.videos.api import refresh_queue
+from modules.videos.channel.lib import spread_channel_downloads, delete_channel
from modules.videos.lib import upsert_video
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.dates import now
from wrolpi.db import get_db_session
+from wrolpi.downloader import download_manager, Download
from wrolpi.errors import UnknownFile
from wrolpi.root_api import api_app
-from wrolpi.test.common import wrap_test_db, get_all_messages_in_queue, TestAPI
+from wrolpi.test.common import wrap_test_db, TestAPI
class TestVideoAPI(TestAPI):
@@ -39,83 +42,83 @@ class TestVideoAPI(TestAPI):
@wrap_test_db
def test_channel(self):
- with tempfile.TemporaryDirectory() as media_dir, \
- mock.patch('wrolpi.common.get_media_directory', lambda: pathlib.Path(media_dir)):
- channel_directory = tempfile.TemporaryDirectory(dir=media_dir).name
- pathlib.Path(channel_directory).mkdir()
- new_channel = dict(
- directory=channel_directory,
- match_regex='asdf',
- name='Example Channel 1',
- url='https://example.com/channel1',
- )
+ channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name
+ channel_directory_name = channel_directory.split('/')[-1]
+ pathlib.Path(channel_directory).mkdir()
- # Channel doesn't exist
- request, response = api_app.test_client.get('/api/videos/channels/examplechannel1')
- assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
+ new_channel = dict(
+ directory=channel_directory,
+ match_regex='asdf',
+ name='Example Channel 1',
+ url='https://example.com/channel1',
+ )
- # Create it
- request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
- assert response.status_code == HTTPStatus.CREATED
- location = response.headers['Location']
- request, response = api_app.test_client.get(location)
- created = response.json['channel']
- self.assertIsNotNone(created)
- self.assertIsNotNone(created['id'])
+ # Channel doesn't exist
+ request, response = api_app.test_client.get('/api/videos/channels/examplechannel1')
+ assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
- # Channel directory should be relative to the media directory
- assert not pathlib.Path(created['directory']).is_absolute(), \
- f'Channel directory is absolute: {created["directory"]}'
+ # Create it
+ request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.CREATED
+ location = response.headers['Location']
+ request, response = api_app.test_client.get(location)
+ created = response.json['channel']
+ self.assertIsNotNone(created)
+ self.assertIsNotNone(created['id'])
- # Get the link that was computed
- new_channel['link'] = response.json['channel']['link']
- assert new_channel['link']
+ # Channel directory should be relative to the media directory
+ assert not pathlib.Path(created['directory']).is_absolute(), \
+ f'Channel directory is absolute: {created["directory"]}'
- # Can't create it again
- request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
- assert response.status_code == HTTPStatus.BAD_REQUEST
+ # Get the link that was computed
+ new_channel['link'] = response.json['channel']['link']
+ assert new_channel['link']
- # Update it
- new_channel['name'] = 'Example Channel 2'
- new_channel['directory'] = str(new_channel['directory'])
- request, response = api_app.test_client.put(location, data=json.dumps(new_channel))
- assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
- request, response = api_app.test_client.get(location)
- assert response.status_code == HTTPStatus.OK
- self.assertDictContains(response.json['channel'], {
- 'id': 1,
- 'name': 'Example Channel 2',
- 'directory': channel_directory,
- 'match_regex': 'asdf',
- 'url': 'https://example.com/channel1',
- })
+ # Can't create it again
+ request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.BAD_REQUEST
- # Patch it
- patch = {'name': 'new name'}
- request, response = api_app.test_client.patch(location, data=json.dumps(patch))
- assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
- request, response = api_app.test_client.get(location)
- assert response.status_code == HTTPStatus.OK
- self.assertDictContains(response.json['channel'], {
- 'id': 1,
- 'name': 'new name',
- 'directory': channel_directory,
- 'match_regex': 'asdf',
- 'url': 'https://example.com/channel1',
- })
+ # Update it
+ new_channel['name'] = 'Example Channel 2'
+ new_channel['directory'] = str(new_channel['directory'])
+ request, response = api_app.test_client.put(location, data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
+ request, response = api_app.test_client.get(location)
+ assert response.status_code == HTTPStatus.OK
+ self.assertDictContains(response.json['channel'], {
+ 'id': 1,
+ 'name': 'Example Channel 2',
+ 'directory': channel_directory_name,
+ 'match_regex': 'asdf',
+ 'url': 'https://example.com/channel1',
+ })
- # Can't update channel that doesn't exist
- request, response = api_app.test_client.put('/api/videos/channels/doesnt_exist',
- data=json.dumps(new_channel))
- assert response.status_code == HTTPStatus.NOT_FOUND
+ # Patch it
+ patch = {'name': 'new name'}
+ request, response = api_app.test_client.patch(location, data=json.dumps(patch))
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
+ request, response = api_app.test_client.get(location)
+ assert response.status_code == HTTPStatus.OK
+ self.assertDictContains(response.json['channel'], {
+ 'id': 1,
+ 'name': 'new name',
+ 'directory': channel_directory_name,
+ 'match_regex': 'asdf',
+ 'url': 'https://example.com/channel1',
+ })
- # Delete the new channel
- request, response = api_app.test_client.delete(location)
- assert response.status_code == HTTPStatus.NO_CONTENT
+ # Can't update channel that doesn't exist
+ request, response = api_app.test_client.put('/api/videos/channels/doesnt_exist',
+ data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.NOT_FOUND
- # Cant delete it again
- request, response = api_app.test_client.delete(location)
- assert response.status_code == HTTPStatus.NOT_FOUND
+ # Delete the new channel
+ request, response = api_app.test_client.delete(location)
+ assert response.status_code == HTTPStatus.NO_CONTENT
+
+ # Cant delete it again
+ request, response = api_app.test_client.delete(location)
+ assert response.status_code == HTTPStatus.NOT_FOUND
@wrap_test_db
def test_channel_conflicts(self):
@@ -210,15 +213,61 @@ class TestVideoAPI(TestAPI):
assert response.status_code == HTTPStatus.CREATED, response.json
assert location != response.headers['Location']
+ @wrap_test_db
+ def test_refresh_no_channel(self):
+ with get_db_session() as session:
+ videos_path = pathlib.Path(self.tmp_dir.name) / 'videos'
+ videos_path.mkdir()
+
+ # A regular user-defined channel
+ channel1_path = videos_path / 'channel1'
+ channel1_path.mkdir()
+ vid1 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
+ vid1.touch()
+ vid2 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
+ vid2.touch()
+
+ channel1 = Channel(name='channel1', link='channel1', directory=channel1_path)
+ session.add(channel1)
+ session.commit()
+
+ # The special NO CHANNEL directory.
+ from modules.videos.downloader import get_no_channel_directory
+ no_channel_path = get_no_channel_directory()
+ no_channel_path.mkdir()
+ vid3 = no_channel_path / 'some video.mp4'
+ vid3.touch()
+ no_channel_subdir = no_channel_path / 'subdir'
+ no_channel_subdir.mkdir()
+ vid4 = no_channel_subdir / 'other video.mp4'
+ vid4.touch()
+
+ api_app.test_client.post('/api/videos/refresh')
+
+ videos = session.query(Video).order_by(Video.title).all()
+ self.assertEqual(len(videos), 4)
+
+ # Add some fake videos, they should be removed.
+ session.add(Video(title='vid5'))
+ session.add(Video(title='vid6', channel_id=channel1.id))
+ session.commit()
+
+ videos = session.query(Video).order_by(Video.title).all()
+ self.assertEqual(len(videos), 6)
+
+ api_app.test_client.post('/api/videos/refresh')
+
+ videos = session.query(Video).order_by(Video.title).all()
+ self.assertEqual(len(videos), 4)
+
@wrap_test_db
def test_refresh_videos(self):
# There should be no messages until a refresh is called.
pytest.raises(Empty, refresh_queue.get_nowait)
# Setup a fake channel directory.
- with get_db_session() as session, \
- tempfile.TemporaryDirectory() as channel_dir:
- channel_path = pathlib.Path(channel_dir)
+ with get_db_session() as session:
+ channel_path = pathlib.Path(self.tmp_dir.name)
# Files in subdirectories should be found and handled properly.
subdir = channel_path / 'subdir'
@@ -243,26 +292,26 @@ class TestVideoAPI(TestAPI):
poster2.touch()
# Create a channel, associate videos with it.
- channel = Channel(directory=channel_dir, link='foo', name='foo')
+ channel = Channel(directory=channel_path, link='foo', name='foo')
session.add(channel)
session.flush()
session.refresh(channel)
video1 = upsert_video(session, vid1, channel)
video2 = upsert_video(session, vid2, channel)
session.commit()
- self.assertEqual({i.video_path for i in channel.videos},
- {pathlib.Path('subdir/' + vid1.name), pathlib.Path(vid2.name)})
+ self.assertEqual({str(i.video_path.path) for i in channel.videos},
+ {f'{channel_path}/subdir/{vid1.name}', f'{channel_path}/{vid2.name}'})
# Poster files were found.
- self.assertEqual(video1.poster_path, pathlib.Path('subdir/' + poster1.name))
- self.assertEqual(video2.poster_path, pathlib.Path(poster2.name))
+ self.assertEqual(video1.poster_path.path, subdir / poster1.name)
+ self.assertEqual(video2.poster_path.path, channel_path / poster2.name)
# Add a bogus file, this should be removed during the refresh
- self.assertNotIn('foo', {i.video_path for i in channel.videos})
+ self.assertNotIn('foo', {i.video_path.path for i in channel.videos})
session.add(Video(video_path='foo', channel_id=channel.id))
session.flush()
session.refresh(channel)
- self.assertIn('foo', {str(i.video_path) for i in channel.videos})
+ self.assertIn(f'{self.tmp_dir.name}/foo', {str(i.video_path.path) for i in channel.videos})
self.assertEqual(len(channel.videos), 3)
# Add a video that isn't in the DB, it should be found and any meta files associated with it
@@ -277,10 +326,10 @@ class TestVideoAPI(TestAPI):
# Finally, call the refresh. Again, it should remove the "foo" video, then discover this 3rd video
# file and it's description.
- api_app.test_client.post('/api/videos:refresh')
+ api_app.test_client.post('/api/videos/refresh')
# Bogus file was removed
- self.assertNotIn('foo', {i.video_path for i in channel.videos})
+ self.assertNotIn(f'{self.tmp_dir.name}/foo', {i.video_path.path for i in channel.videos})
# Final channel video list we built
expected = {
@@ -290,7 +339,7 @@ class TestVideoAPI(TestAPI):
}
def str_or_none(i):
- return str(i) if i else None
+ return str(i.__json__()) if i else None
self.assertEqual(
{(str_or_none(i.video_path), str_or_none(i.poster_path), str_or_none(i.description_path))
@@ -300,10 +349,6 @@ class TestVideoAPI(TestAPI):
assert poster3.is_file(), 'Orphan jpg file was deleted!'
- # During the refresh process, messages are pushed to a queue, make sure there are messages there
- messages = get_all_messages_in_queue(refresh_queue)
- assert 'refresh-started' in [i.get('code') for i in messages]
-
@wrap_test_db
def test_get_channel_videos(self):
with get_db_session(commit=True) as session:
@@ -466,3 +511,93 @@ class TestVideoAPI(TestAPI):
response = do_search('b', 2)
assert [i['id'] for i in response.json['videos']] == [1, 2]
assert response.json['totals']['videos'] == 4
+
+ @wrap_test_db
+ def test_channel_no_download_frequency(self):
+ """
+ A channel does not require a download frequency.
+ """
+ channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name
+ os.mkdir(channel_directory)
+
+ new_channel = dict(
+ directory=channel_directory,
+ name='Example Channel 1',
+ url='https://example.com/channel1',
+ download_frequency=None,
+ )
+
+ # Create the Channel
+ request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.CREATED
+ location = response.headers['Location']
+
+ # No downloads are scheduled.
+ spread_channel_downloads()
+ with get_db_session() as session:
+ downloads = list(download_manager.get_new_downloads(session))
+ self.assertEqual(len(downloads), 0)
+
+ # Get the Channel
+ request, response = api_app.test_client.get(location)
+ channel = response.json['channel']
+ self.assertEqual(channel['download_frequency'], None)
+
+ # Update the Channel with a frequency.
+ new_channel = dict(
+ directory=channel['directory'],
+ name=channel['name'],
+ url=channel['url'],
+ download_frequency=10,
+ )
+ request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}',
+ data=json.dumps(new_channel))
+ self.assertEqual(response.status_code, 204, response.json)
+
+ # Remove the frequency.
+ new_channel = dict(
+ directory=channel['directory'],
+ name=channel['name'],
+ url=channel['url'],
+ download_frequency=None,
+ )
+ request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}',
+ data=json.dumps(new_channel))
+ self.assertEqual(response.status_code, 204, response.json)
+
+
+def test_channel_frequency_update(download_channel, test_client, test_session):
+ """
+ A Channel's Download record is updated when the Channel's frequency is updated.
+ """
+ old_frequency = download_channel.get_download().frequency
+ assert old_frequency
+
+ data = dict(
+ directory=str(download_channel.directory.path),
+ name=download_channel.name,
+ url=download_channel.url,
+ download_frequency=100,
+ )
+ request, response = test_client.put(f'/api/videos/channels/{download_channel.link}', json=data)
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.json
+
+ download = download_channel.get_download()
+ assert download.frequency == 100
+
+ # Only one download
+ downloads = test_session.query(Download).all()
+ assert len(list(downloads)) == 1
+
+
+def test_delete_channel_delete_download(download_channel, test_session):
+ """
+ Deleting a Channel deletes it's Download.
+ """
+ downloads = test_session.query(Download).all()
+ assert all(i.url == download_channel.url for i in downloads)
+
+ delete_channel(download_channel.link)
+
+ downloads = test_session.query(Download).all()
+ assert not list(downloads)
diff --git a/modules/videos/channel/test/test_channel.py b/modules/videos/channel/test/test_channel.py
deleted file mode 100644
index 34b3d695..00000000
--- a/modules/videos/channel/test/test_channel.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import pathlib
-import unittest
-
-from modules.videos.models import Channel
-from modules.videos.test.common import create_channel_structure
-from wrolpi.db import get_db_session
-from wrolpi.test.common import wrap_test_db
-
-
-class TestChannel(unittest.TestCase):
-
- @wrap_test_db
- @create_channel_structure(
- {
- 'Foo': ['vid1.mp4'],
- }
- )
- def test_channel(self, tempdir):
- with get_db_session() as session:
- channel = session.query(Channel).one()
- self.assertIsInstance(channel.directory, pathlib.Path)
diff --git a/modules/videos/common.py b/modules/videos/common.py
index 284fca4f..a8df39b1 100644
--- a/modules/videos/common.py
+++ b/modules/videos/common.py
@@ -12,11 +12,12 @@ from PIL import Image
from sqlalchemy.orm import Session
from wrolpi import before_startup
-from wrolpi.common import sanitize_link, logger, CONFIG_PATH, get_config, iterify, chunks, get_media_directory, \
- get_absolute_media_path, minimize_dict
+from wrolpi.common import logger, CONFIG_PATH, get_config, iterify, chunks, get_media_directory, \
+ minimize_dict
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \
- ChannelLinkConflict, ChannelDirectoryConflict
+ ChannelLinkConflict, ChannelDirectoryConflict, ChannelSourceIdConflict
+from wrolpi.media_path import MediaPath
from wrolpi.vars import DOCKERIZED, DEFAULT_FILE_PERMISSIONS
from .models import Channel, Video
@@ -24,7 +25,7 @@ logger = logger.getChild(__name__)
REQUIRED_OPTIONS = ['name', 'directory']
MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count', 'link'}
-MINIMUM_INFO_JSON_KEYS = {'description', 'view_count'}
+MINIMUM_INFO_JSON_KEYS = {'description', 'view_count', 'webpage_url'}
MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'channel_id', 'favorite', 'size',
'poster_path', 'caption_path', 'video_path', 'info_json', 'channel', 'viewed', 'source_id',
'view_count'}
@@ -67,35 +68,38 @@ def import_videos_config():
config = load_channels_config()
with get_db_session(commit=True) as session:
- for section in config:
- for option in (i for i in REQUIRED_OPTIONS if i not in config[section]):
- raise ConfigError(f'Channel "{section}" is required to have "{option}"')
+ for link in config:
+ for option in (i for i in REQUIRED_OPTIONS if i not in config[link]):
+ raise ConfigError(f'Channel "{link}" is required to have "{option}"')
- name, directory = config[section]['name'], config[section]['directory']
+ name, directory = config[link]['name'], config[link]['directory']
- link = sanitize_link(name)
- matches = session.query(Channel).filter(Channel.link == link)
- if not matches.count():
- # Channel not yet in the DB, add it
+ channel = session.query(Channel).filter(Channel.link == link).one_or_none()
+ if not channel:
+ # Channel not yet in the DB, add it.
channel = Channel(link=link)
- else:
- channel = matches.one()
# Only name and directory are required
channel.name = name
channel.directory = str(directory)
- channel.calculate_duration = config[section].get('calculate_duration')
- channel.download_frequency = config[section].get('download_frequency')
- channel.generate_posters = config[section].get('generate_posters')
- channel.match_regex = config[section].get('match_regex')
- channel.skip_download_videos = list(set(config[section].get('skip_download_videos', {})))
- channel.url = config[section].get('url')
+ channel.calculate_duration = config[link].get('calculate_duration') or channel.calculate_duration
+ channel.download_frequency = config[link].get('download_frequency') or channel.download_frequency
+ channel.generate_posters = config[link].get('generate_posters') or channel.generate_posters
+ channel.match_regex = config[link].get('match_regex') or channel.match_regex
+ channel.skip_download_videos = list(set(config[link].get('skip_download_videos', {})))
+ channel.source_id = config[link].get('source_id') or channel.source_id
+ channel.url = config[link].get('url') or channel.url
+
+ if not channel.source_id and channel.url:
+ # If we can download from a channel, we must have its source_id.
+ from .downloader import get_channel_source_id
+ channel.source_id = get_channel_source_id(channel.url)
session.add(channel)
# Set favorite Videos of this Channel.
- favorites = config[section].get('favorites', {})
+ favorites = config[link].get('favorites', {})
if favorites:
videos = session.query(Video).filter(Video.source_id.in_(favorites.keys()))
for video in videos:
@@ -107,13 +111,12 @@ def import_videos_config():
VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'}
-def get_absolute_video_path(video: Video, kind: str = 'video') -> Path:
+def get_absolute_video_path(video: Video, kind: str = 'video') -> MediaPath:
if kind not in VALID_VIDEO_KINDS:
raise Exception(f'Unknown video path kind {kind}')
- directory = get_absolute_media_path(video.channel.directory)
path = getattr(video, f'{kind}_path')
- if directory and path:
- return directory / path
+ if path:
+ return path
raise UnknownFile()
@@ -123,7 +126,7 @@ get_absolute_video_description = partial(get_absolute_video_path, kind='descript
get_absolute_video_info_json = partial(get_absolute_video_path, kind='info_json')
-def get_absolute_video_files(video: Video) -> List[Path]:
+def get_absolute_video_files(video: Video) -> List[MediaPath]:
"""
Get all video files that exist.
"""
@@ -173,7 +176,7 @@ def any_extensions(filename: str, extensions: Iterable = ()):
match_video_extensions = partial(any_extensions, extensions=VIDEO_EXTENSIONS)
-def generate_video_paths(directory: Union[str, pathlib.Path], relative_to=None) -> Tuple[str, pathlib.Path]:
+def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]:
"""
Generate a list of video paths in the provided directory.
"""
@@ -182,10 +185,7 @@ def generate_video_paths(directory: Union[str, pathlib.Path], relative_to=None)
for child in directory.iterdir():
if child.is_file() and match_video_extensions(child.name):
child = child.absolute()
- if relative_to:
- yield child.relative_to(relative_to)
- else:
- yield child
+ yield child
elif child.is_dir():
yield from generate_video_paths(child)
@@ -232,7 +232,8 @@ def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]:
yield sorted(paths)[0]
-def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, link=None, directory=None):
+def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, link=None, directory=None,
+ source_id=None):
"""
Search for any channels that conflict with the provided args, raise a relevant exception if any conflicts are found.
"""
@@ -263,6 +264,10 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None,
conflicts = base_where.filter(Channel.directory == directory)
if list(conflicts):
raise ChannelDirectoryConflict()
+ if source_id:
+ conflicts = base_where.filter(Channel.source_id == source_id)
+ if list(conflicts):
+ raise ChannelSourceIdConflict()
def verify_config():
@@ -295,10 +300,14 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
"""
path = str(path)
+ ignored_directories = {
+ str(get_media_directory() / 'videos/NO CHANNEL'),
+ }
+
if os.path.isdir(path):
# The provided path is a directory, return it's subdirectories, or itself if no subdirectories exist
paths = [os.path.join(path, i) for i in os.listdir(path)]
- paths = sorted(i for i in paths if os.path.isdir(i))
+ paths = sorted(i for i in paths if os.path.isdir(i) and i not in ignored_directories)
if len(paths) == 0:
return [path]
return paths
@@ -307,7 +316,8 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
paths = os.listdir(head)
paths = [os.path.join(head, i) for i in paths]
pattern = path.lower()
- paths = sorted(i for i in paths if os.path.isdir(i) and i.lower().startswith(pattern))
+ paths = sorted(
+ i for i in paths if os.path.isdir(i) and i.lower().startswith(pattern) and i not in ignored_directories)
return paths
@@ -354,14 +364,12 @@ async def generate_bulk_posters(video_ids: List[int]):
with get_db_session(commit=True) as session:
videos = session.query(Video).filter(Video.id.in_(video_ids))
for video in videos:
- video_path = get_absolute_video_path(video)
+ video_path = video.video_path.path
poster_path = replace_extension(video_path, '.jpg')
if not poster_path.exists():
generate_video_poster(video_path)
- channel_dir = get_absolute_media_path(video.channel.directory)
- poster_path = poster_path.relative_to(channel_dir)
- video.poster_path = str(poster_path)
+ video.poster_path = poster_path
logger.info('Done generating video posters')
@@ -400,10 +408,9 @@ def bulk_validate_posters(video_ids: List[int]):
for video_ids in chunks(video_ids, 10):
for video_id in video_ids:
with get_db_session(commit=True) as session:
- video = session.query(Video).filter_by(id=video_id).one()
- channel = video.channel
+ video: Video = session.query(Video).filter_by(id=video_id).one()
- poster_path: Path = get_absolute_video_poster(video)
+ poster_path: Path = video.poster_path.path
new_poster_path = poster_path.with_suffix('.jpg')
if poster_path != new_poster_path and new_poster_path.exists():
@@ -423,10 +430,8 @@ def bulk_validate_posters(video_ids: List[int]):
else:
logger.debug(f'Poster was already valid: {new_poster_path}')
- channel_dir = get_absolute_media_path(channel.directory)
-
# Update the video with the new poster path. Mark it as validated.
- video.poster_path = str(new_poster_path.relative_to(channel_dir))
+ video.poster_path = new_poster_path
video.validated_poster = True
logger.info('Done validating video posters.')
@@ -465,6 +470,10 @@ async def get_bulk_video_info_json(video_ids: List[int]):
video = session.query(Video).filter_by(id=video_id).one()
logger.debug(f'Getting video info_json data: {video}')
+ if not video.video_path:
+ logger.warning(f'Refusing to get info_json for video without a video file: {video}')
+ continue
+
try:
info_json = video.get_info_json()
if info_json:
@@ -475,8 +484,7 @@ async def get_bulk_video_info_json(video_ids: List[int]):
video.duration = info_json.get('duration')
elif not video.duration:
# As a last resort, get duration from the video file.
- video_path = get_absolute_video_path(video)
- video.duration = get_video_duration(video_path)
+ video.duration = get_video_duration(video.video_path.path)
except Exception:
logger.warning(f'Unable to get meta data of {video}', exc_info=True)
@@ -493,16 +501,15 @@ async def get_bulk_video_size(video_ids: List[int]):
for video_id in video_ids:
video = session.query(Video).filter_by(id=video_id).one()
logger.debug(f'Getting video size: {video.id} {video.video_path}')
- video_path = get_absolute_video_path(video)
- size = video_path.stat().st_size
+ size = video.video_path.path.stat().st_size
video.size = size
logger.info('Done getting video sizes')
def update_view_count(channel_id: int):
"""
- Update view_count for all Videos in a channel.
+ Update view_count for all Videos in a channel using it's info_json file.
"""
with get_db_session() as session:
channel = session.query(Channel).filter_by(id=channel_id).one()
diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py
new file mode 100644
index 00000000..a4bdd9d0
--- /dev/null
+++ b/modules/videos/conftest.py
@@ -0,0 +1,33 @@
+import pytest
+
+from modules.videos.channel.lib import spread_channel_downloads
+from modules.videos.models import Channel
+from wrolpi.downloader import DownloadFrequency
+
+
+@pytest.fixture
+def simple_channel(test_session, test_directory) -> Channel:
+ """
+ Get a Channel with the minimum properties. This Channel has no download!
+ """
+ channel = Channel(
+ directory=test_directory,
+ name='Example Channel 1',
+ url='https://example.com/channel1',
+ download_frequency=None, # noqa
+ link='examplechannel1',
+ )
+ test_session.add(channel)
+ test_session.commit()
+ return channel
+
+
+@pytest.fixture
+def download_channel(simple_channel) -> Channel:
+ """
+ Get a test Channel that has a download frequency.
+ """
+ # Add a frequency to the test channel, then give it a download.
+ simple_channel.download_frequency = DownloadFrequency.weekly
+ spread_channel_downloads()
+ return simple_channel
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index b7e65fe9..4663ff53 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -1,21 +1,27 @@
#! /usr/bin/env python3
+import json
import pathlib
import re
-from queue import Queue
-from random import shuffle
-from typing import Tuple, List, Dict
+from abc import ABC
+from typing import Tuple, List
-from sqlalchemy import or_
from sqlalchemy.orm.exc import NoResultFound
-from youtube_dl import YoutubeDL
+from yt_dlp import YoutubeDL
+from yt_dlp.extractor import YoutubeTabIE
+from yt_dlp.utils import UnsupportedError, DownloadError
-from wrolpi.common import logger, ProgressReporter, run_after, get_absolute_media_path
-from wrolpi.dates import now, today
+from wrolpi.common import logger, sanitize_link, extract_domain, \
+ get_media_directory
+from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
-from wrolpi.errors import UnknownChannel, ChannelURLEmpty
+from wrolpi.downloader import Downloader, Download
+from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
+from wrolpi.vars import PYTEST
+from .channel.lib import create_channel, get_channel
from .common import load_downloader_config, update_view_count
-from .lib import save_channels_config, upsert_video
+from .lib import upsert_video, _refresh_videos
from .models import Video, Channel
+from .video_url_resolver import video_url_resolver
logger = logger.getChild(__name__)
ydl_logger = logger.getChild('youtube-dl')
@@ -24,17 +30,219 @@ YDL = YoutubeDL()
YDL.params['logger'] = ydl_logger
YDL.add_default_info_extractors()
+# Channels are handled differently than a single video.
+ChannelIEs = {
+ YoutubeTabIE,
+}
-def update_channel(channel=None, link: str = None):
+PREFERRED_VIDEO_EXTENSION = 'mp4'
+PREFERRED_VIDEO_FORMAT = 'best[height=720]'
+
+
+def get_no_channel_directory():
+ return get_media_directory() / 'videos/NO CHANNEL'
+
+
+class ChannelDownloader(Downloader, ABC):
+ """
+ Handling downloading of all Videos in a Channel.
+ """
+
+ @staticmethod
+ def valid_url(url) -> bool:
+ for ie in ChannelIEs:
+ if ie.suitable(url):
+ return True
+ return False
+
+ def do_download(self, download: Download):
+ """
+ Update a Channel's catalog, then schedule downloads of every missing video.
+ """
+ with get_db_session() as session:
+ channel = session.query(Channel).filter_by(url=download.url).one()
+
+ update_channel(channel)
+
+ domain = extract_domain(download.url)
+ missing_videos = find_all_missing_videos(channel.link)
+ with get_db_session(commit=True) as session:
+ for video_id, source_id, missing_video in missing_videos:
+ url = video_url_resolver(domain, missing_video)
+ # Schedule any missing videos for download.
+ self.manager.create_download(url, session, skip_download=True)
+
+ if PYTEST:
+ self.manager.do_downloads_sync()
+ else:
+ self.manager.start_downloads()
+
+ return True
+
+
+class VideoDownloader(Downloader, ABC):
+ """
+ Download a single video. Store the video in it's channel's directory, otherwise store it in `videos/NO CHANNEL`.
+ """
+
+ @staticmethod
+ def valid_url(url) -> bool:
+ """
+ Match against all Youtube-DL Info Extractors, except those that match a Channel.
+ """
+ for ie in YDL._ies.values():
+ if ie.suitable(url) and not ChannelDownloader.valid_url(url):
+ try:
+ YDL.extract_info(url, download=False, process=False)
+ return True
+ except UnsupportedError:
+ logger.debug(f'Video downloader extract_info failed for {url}')
+ return False
+ except DownloadError:
+ logger.debug(f'Video downloader extract_info failed for {url}')
+ return False
+ return False
+
+ def do_download(self, download: Download):
+ if download.attempts >= 10:
+ raise UnrecoverableDownloadError('Max download attempts reached')
+
+ url = download.url
+ try:
+ info = YDL.extract_info(url, download=False, process=False)
+ except UnsupportedError as e:
+ # Video wasn't really valid... probably the GenericIE.
+ raise UnrecoverableDownloadError() from e
+
+ channel_name = info.get('channel')
+ channel_id = info.get('channel_id')
+ channel = None
+ if channel_name or channel_id:
+ channel_url = info.get('channel_url')
+ channel = get_or_create_channel(channel_id, None, channel_url, channel_name)
+
+ # Use the default directory if this video has no channel.
+ out_dir = get_no_channel_directory()
+ if channel:
+ out_dir = channel.directory.path
+ out_dir.mkdir(exist_ok=True)
+
+ try:
+ video_path, entry = self.prepare_filename(url, out_dir)
+ # Do the real download.
+ file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
+ cmd = [
+ 'yt-dlp',
+ '-cw', # Continue downloads, do not clobber existing files.
+ '-f', PREFERRED_VIDEO_FORMAT,
+ '--write-subs',
+ '--write-auto-subs',
+ '--write-thumbnail',
+ '--write-info-json',
+ '--merge-output-format', PREFERRED_VIDEO_EXTENSION,
+ '-o', file_name_format,
+ url,
+ ]
+ return_code = self.process_runner(url, cmd, out_dir)
+
+ if return_code != 0:
+ raise ValueError(f'youtube-dl exited with {return_code}')
+
+ if not video_path.is_file():
+ raise ValueError(f'Video file could not be found! {video_path}')
+ except UnrecoverableDownloadError:
+ raise
+ except Exception as e:
+ logger.warning(f'Failed to download "{info["title"]}"', exc_info=e)
+ if _skip_download(e):
+ # The video failed to download, and the error will never be fixed. Skip it forever.
+ source_id = info.get('id')
+ logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
+ f'attempt to download it again.')
+
+ with get_db_session(commit=True) as session:
+ channel = session.query(Channel).filter_by(id=channel.id).one()
+ channel.add_video_to_skip_list(source_id)
+ return
+
+ with get_db_session(commit=True) as session:
+ # If the video is from a channel, it will already be in the database.
+ source_id = entry['id']
+ existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none()
+ id_ = existing_video.id if existing_video else None
+
+ upsert_video(session, video_path, channel, id_=id_)
+
+ # Successful download.
+ return True
+
+ @staticmethod
+ def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
+ if not out_dir.is_dir():
+ raise ValueError(f'Output directory does not exist! {out_dir=}')
+
+ # YoutubeDL expects specific options, add onto the default options
+ config = load_downloader_config()
+ options = config.copy()
+ options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}'
+ # We don't need format when getting the filename.
+ del options['format']
+
+ logger.debug(f'Downloading {url} to {out_dir}')
+
+ # Create a new YoutubeDL for the output directory.
+ ydl = YoutubeDL(options)
+ ydl.params['logger'] = ydl_logger
+ ydl.add_default_info_extractors()
+
+ # Perform the real download. This saves multiple files.
+ entry = ydl.extract_info(url, download=False, process=True)
+ final_filename = pathlib.Path(ydl.prepare_filename(entry))
+ return final_filename, entry
+
+
+channel_downloader = ChannelDownloader()
+# Videos may match the ChannelDownloader, give it a higher priority.
+video_downloader = VideoDownloader(40)
+
+
+def get_or_create_channel(source_id: str = None, link: str = None, url: str = None, name: str = None) -> Channel:
+ """
+ Attempt to find a Channel using the provided params. The params are in order of reliability.
+
+ Creates a new Channel if one cannot be found.
+ """
+ if not link and name:
+ link = sanitize_link(name)
+
+ try:
+ channel = get_channel(link=link, source_id=source_id, url=url, return_dict=False)
+ return channel
+ except UnknownChannel:
+ pass
+
+ channel_directory = get_media_directory() / f'videos/{name}'
+ data = dict(
+ name=name,
+ url=url,
+ link=link,
+ domain=extract_domain(url),
+ directory=str(channel_directory),
+ source_id=source_id,
+ )
+ channel = create_channel(data, return_dict=False)
+ # Create the directory now that the channel is approved.
+ channel_directory.mkdir(exist_ok=True)
+
+ return channel
+
+
+def update_channel(channel: Channel):
"""
Connect to the Channel's host website and pull a catalog of all videos. Insert any new videos into the DB.
It is expected that any missing videos will be downloaded later.
"""
- with get_db_session() as session:
- if not channel:
- channel = session.query(Channel).filter_by(link=link).one()
-
logger.info(f'Downloading video list for {channel.name} at {channel.url} This may take several minutes.')
info = YDL.extract_info(channel.url, download=False, process=False)
if 'url' in info:
@@ -63,16 +271,19 @@ def update_channel(channel=None, link: str = None):
logger.warning(f'entries: {entries}')
raise KeyError('No id key for entry!') from e
+ # In order to store the Video's URL, we will need a quick lookup.
+ urls = {i['id']: i.get('webpage_url') for i in entries}
+
with get_db_session(commit=True) as session:
# Get the channel in this new context.
channel = session.query(Channel).filter_by(id=channel.id).one()
channel.info_json = info
channel.info_date = now()
- channel.increment_next_download()
+ channel.source_id = info.get('id')
with get_db_curs() as curs:
- # Insert any new videos.
+ # Get all known videos in this channel.
query = 'SELECT source_id FROM video WHERE channel_id=%s AND source_id IS NOT NULL'
curs.execute(query, (channel.id,))
known_source_ids = {i[0] for i in curs.fetchall()}
@@ -82,59 +293,19 @@ def update_channel(channel=None, link: str = None):
logger.info(f'Got {len(new_source_ids)} new videos for channel {channel.name}')
channel_id = channel.id
for source_id in new_source_ids:
- session.add(Video(source_id=source_id, channel_id=channel_id))
+ url = urls.get(source_id)
+ session.add(Video(source_id=source_id, channel_id=channel_id, url=url))
+
+ # Write the Channel's info to a JSON file.
+ if channel.directory:
+ info_json_path = channel.directory.path / f'{channel.name}.info.json'
+ with info_json_path.open('wt') as fh:
+ json.dump(info, fh, indent=2)
# Update all view counts using the latest from the Channel's info_json.
update_view_count(channel_id)
-def update_channels(reporter: ProgressReporter, link: str = None):
- """Update all information for each channel. (No downloads performed)"""
-
- with get_db_session() as session:
- if session.query(Channel).count() == 0:
- raise UnknownChannel('No channels exist yet')
-
- if link:
- try:
- channel = session.query(Channel).filter_by(link=link).one()
- except NoResultFound:
- raise UnknownChannel(f'No channel with link: {link}')
- channels = [channel, ]
- else:
- channels = session.query(Channel).filter(
- Channel.url != None, # noqa
- Channel.url != '',
- or_(
- Channel.next_download == None, # noqa
- Channel.next_download <= today(),
- )
- ).all()
-
- if len(channels) == 0:
- logger.warning(f'All channels are up to date')
-
- reporter.set_progress_total(0, len(channels))
- reporter.send_progress(0, 0, f'{len(channels)} channels scheduled for update')
-
- # Randomize downloading of channels.
- shuffle(channels)
-
- logger.debug(f'Getting info for {len(channels)} channels')
- for idx, channel in enumerate(channels):
- reporter.send_progress(0, idx, f'Getting video list for {channel.name}')
- try:
- update_channel(channel)
- except Exception:
- logger.critical(f'Unable to get video list for {channel.name}', exc_info=True)
- continue
-
- if channels:
- reporter.send_progress(0, len(channels), 'Done downloading video lists')
- else:
- reporter.finish(0, 'Done downloading video lists')
-
-
def _find_all_missing_videos(link: str = None) -> List[Tuple]:
"""
Get all Video entries which don't have the required media files (i.e. hasn't been downloaded). Restrict to a
@@ -162,11 +333,12 @@ def _find_all_missing_videos(link: str = None) -> List[Tuple]:
WHERE
channel.url IS NOT NULL
AND channel.url != ''
- AND source_id IS NOT NULL
+ AND video.source_id IS NOT NULL
{where}
- AND channel_id IS NOT NULL
+ AND video.channel_id IS NOT NULL
AND (video_path IS NULL OR video_path = '' OR poster_path IS NULL OR poster_path = '')
'''
+ logger.debug(query)
curs.execute(query, params)
missing_videos = list(curs.fetchall())
return missing_videos
@@ -193,6 +365,27 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
# Get all channels while in the db context.
channels = list(channels)
+ # Check that each channel has some videos. We can't be sure what is missing if we don't know what we have.
+ for channel in channels:
+ refresh = False
+ with get_db_curs() as curs:
+ curs.execute('''
+ SELECT
+ COUNT(v.id)
+ FROM
+ channel c
+ LEFT OUTER JOIN video v on c.id = v.channel_id
+ WHERE
+ c.id = %s
+ ''', (channel.id,))
+ video_count = curs.fetchall()[0]
+ if video_count == 0:
+ # No videos for this channel. We can't be sure if any videos are missing without a refresh.
+ refresh = True
+ if refresh:
+ # Refresh this channel's videos.
+ _refresh_videos([channel.link])
+
channels = {i.id: i for i in channels}
match_regexen = {i: re.compile(j.match_regex) for i, j in channels.items() if j.match_regex}
@@ -221,31 +414,7 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
match_regex: re.compile = match_regexen.get(channel_id)
if not match_regex or (match_regex and missing_video['title'] and match_regex.match(missing_video['title'])):
# No title match regex, or the title matches the regex.
- yield channel, id_, missing_video
-
-
-def download_video(channel: Channel, video: Dict) -> Tuple[pathlib.Path, Dict]:
- """
- Download a video (and associated posters/etc) to it's channel's directory.
-
- :param channel:
- :param video: A YoutubeDL info entry dictionary
- :return:
- """
- # YoutubeDL expects specific options, add onto the default options
- config = load_downloader_config()
- options = dict(config)
- directory = get_absolute_media_path(channel.directory)
- options['outtmpl'] = f'{directory}/{config["file_name_format"]}'
-
- ydl = YoutubeDL(options)
- ydl.add_default_info_extractors()
- source_id = video['id']
- url = f'https://www.youtube.com/watch?v={source_id}'
- entry = ydl.extract_info(url, download=True, process=True)
- final_filename = ydl.prepare_filename(entry)
- final_filename = pathlib.Path(final_filename)
- return final_filename, entry
+ yield id_, source_id, missing_video
UNRECOVERABLE_ERRORS = {
@@ -268,42 +437,6 @@ def _skip_download(error):
return False
-@run_after(save_channels_config)
-def download_all_missing_videos(reporter: ProgressReporter, link: str = None):
- """Find any videos identified by the info packet that haven't yet been downloaded, download them."""
- missing_videos = list(find_all_missing_videos(link))
- reporter.set_progress_total(1, len(missing_videos))
- reporter.message(1, f'Found {len(missing_videos)} missing videos.')
-
- for idx, (channel, id_, missing_video) in enumerate(missing_videos):
- reporter.send_progress(1, idx, f'Downloading {channel.name}: {missing_video["title"]}')
- try:
- video_path, info_json = download_video(channel, missing_video)
- except Exception as e:
- logger.warning(f'Failed to download "{missing_video["title"]}"', exc_info=e)
- if _skip_download(e):
- # The video failed to download, and the error will never be fixed. Skip it forever.
- source_id = missing_video.get('id')
- logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
- f'attempt to download it again.')
-
- with get_db_session(commit=True) as session:
- channel = session.query(Channel).filter_by(id=channel.id).one()
- channel.add_video_to_skip_list(source_id)
-
- reporter.error(1, f'Failed to download "{missing_video["title"]}", see server logs...')
- continue
-
- with get_db_session(commit=True) as session:
- upsert_video(session, video_path, channel, id_=id_, info_json=info_json)
-
- reporter.finish(1, 'All videos are downloaded')
-
-
-def main(args=None):
- """Find and download any missing videos. Parse any arguments passed by the cmd-line."""
- q = Queue()
- reporter = ProgressReporter(q, 2)
- update_channels(reporter)
- download_all_missing_videos(reporter)
- return 0
+def get_channel_source_id(url: str) -> str:
+ channel_info = YDL.extract_info(url, download=False, process=False)
+ return channel_info['id']
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 64892532..d749644b 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -2,15 +2,15 @@ import asyncio
import json
import pathlib
import re
-from multiprocessing.queues import Queue
from typing import Tuple
from uuid import uuid1
from sqlalchemy.orm import Session
-from wrolpi.common import ProgressReporter, save_settings_config, get_absolute_media_path
-from wrolpi.common import logger
+from wrolpi.common import logger, chunks
+from wrolpi.common import save_settings_config
from wrolpi.db import get_db_curs, get_db_session
+from wrolpi.vars import PYTEST
from .captions import insert_bulk_captions, process_captions
from .common import generate_bulk_posters, get_bulk_video_info_json, get_bulk_video_size, \
generate_video_paths, remove_duplicate_video_paths, bulk_validate_posters, \
@@ -19,6 +19,8 @@ from .models import Channel, Video
logger = logger.getChild(__name__)
+DEFAULT_DOWNLOAD_FREQUENCY = 60 * 60 * 24 * 7 # weekly
+
def refresh_channel_video_captions() -> bool:
with get_db_curs() as curs:
@@ -86,6 +88,7 @@ def refresh_channel_info_json() -> bool:
WHERE
v.video_path IS NOT NULL
AND v.info_json_path IS NOT NULL
+ AND v.info_json_path != ''
AND (v.duration IS NULL OR v.view_count IS NULL)
'''
curs.execute(query)
@@ -128,50 +131,38 @@ def process_video_meta_data():
refresh_channel_calculate_size()
-def refresh_channel_videos(channel: Channel, reporter: ProgressReporter):
+def refresh_channel_videos(channel: Channel):
"""
Find all video files in a channel's directory. Add any videos not in the DB to the DB.
"""
- # This function is hard to predict, so we will simply progress in chunks :(
- reporter.set_progress_total(1, 7)
- reporter.send_progress(1, 0, 'Preparing channel.')
-
# Set the idempotency key so we can remove any videos not touched during this search
with get_db_curs(commit=True) as curs:
curs.execute('UPDATE video SET idempotency=NULL WHERE channel_id=%s', (channel.id,))
- reporter.send_progress(1, 1, 'Finding all videos, checking for duplicates.')
-
idempotency = str(uuid1())
- directory = get_absolute_media_path(channel.directory)
+ directory = channel.directory.path
# A set of absolute paths that exist in the file system
possible_new_paths = generate_video_paths(directory)
possible_new_paths = remove_duplicate_video_paths(possible_new_paths)
- reporter.send_progress(1, 2, 'Matching all videos to the database.')
-
# Update all videos that match the current video paths
- relative_new_paths = [str(i.relative_to(directory)) for i in possible_new_paths]
+ new_paths = [str(i) for i in possible_new_paths]
with get_db_curs(commit=True) as curs:
query = 'UPDATE video SET idempotency = %s WHERE channel_id = %s AND video_path = ANY(%s) RETURNING video_path'
- curs.execute(query, (idempotency, channel.id, relative_new_paths))
+ curs.execute(query, (idempotency, channel.id, new_paths))
existing_paths = {i for (i,) in curs.fetchall()}
- reporter.send_progress(1, 3)
-
# Get the paths for any video not yet in the DB
# (paths in DB are relative, but we need to pass an absolute path)
- new_videos = {p for p in possible_new_paths if str(p.relative_to(directory)) not in existing_paths}
+ new_videos = {i for i in possible_new_paths if str(i) not in existing_paths}
- reporter.send_progress(1, 4, f'Inserting {len(new_videos)} new videos.')
-
- for video_path in new_videos:
+ for chunk in chunks(new_videos, 20):
with get_db_session(commit=True) as session:
- upsert_video(session, pathlib.Path(video_path), channel, idempotency=idempotency)
- logger.debug(f'{channel.name}: Added {video_path}')
-
- reporter.send_progress(1, 5, 'Deleting unnecessary video entries.')
+ for video_path in chunk:
+ video_path = pathlib.Path(video_path)
+ upsert_video(session, video_path, channel, idempotency=idempotency)
+ logger.debug(f'{channel.name}: Added {video_path.name}')
with get_db_curs(commit=True) as curs:
curs.execute('DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL RETURNING id', (channel.id,))
@@ -183,14 +174,50 @@ def refresh_channel_videos(channel: Channel, reporter: ProgressReporter):
logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ')
- reporter.send_progress(1, 6, f'Updating video view counts')
-
update_view_count(channel.id)
- reporter.send_progress(1, 7, f'Processed all videos for {channel.name}')
+
+def refresh_no_channel_videos():
+ """
+ Refresh the Videos in the NO CHANNEL directory.
+ """
+ from modules.videos.downloader import get_no_channel_directory
+ directory = get_no_channel_directory()
+ if not directory.is_dir():
+ return
+
+ logger.info('Refreshing NO CHANNEL videos')
+
+ idempotency = str(uuid1())
+
+ # Get all WROLPi compatible videos, remove any duplicates (different formats).
+ possible_new_paths = generate_video_paths(directory)
+ possible_new_paths = remove_duplicate_video_paths(possible_new_paths)
+
+ new_paths = [str(i) for i in possible_new_paths]
+ with get_db_curs(commit=True) as curs:
+ query = 'UPDATE video SET idempotency = %s WHERE video_path = ANY(%s) RETURNING video_path'
+ curs.execute(query, (idempotency, new_paths))
+ existing_paths = {i for (i,) in curs.fetchall()}
+
+ new_videos = {i for i in possible_new_paths if str(i) not in existing_paths}
+
+ for chunk in chunks(new_videos, 20):
+ with get_db_session(commit=True) as session:
+ for video_path in chunk:
+ upsert_video(session, pathlib.Path(video_path), idempotency=idempotency)
+ logger.debug(f'Added NO CHANNEL video {video_path}')
+
+ with get_db_curs(commit=True) as curs:
+ curs.execute('DELETE FROM video WHERE channel_id IS NULL AND idempotency IS NULL RETURNING id')
+ deleted_count = len(curs.fetchall())
+
+ if deleted_count:
+ deleted_status = f'Deleted {deleted_count} video records in NO CHANNEL.'
+ logger.info(deleted_status)
-def _refresh_videos(q: Queue, channel_links: list = None):
+def _refresh_videos(channel_links: list = None):
"""
Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the
file system.
@@ -201,10 +228,6 @@ def _refresh_videos(q: Queue, channel_links: list = None):
"""
logger.info('Refreshing video files')
with get_db_session() as session:
- reporter = ProgressReporter(q, 2)
- reporter.code('refresh-started')
- reporter.set_progress_total(0, session.query(Channel).count())
-
if channel_links:
channels = session.query(Channel).filter(Channel.link.in_(channel_links))
else:
@@ -218,14 +241,15 @@ def _refresh_videos(q: Queue, channel_links: list = None):
raise Exception(f'No channels in DB. Have you created any?')
for idx, channel in enumerate(channels):
- reporter.send_progress(0, idx, f'Checking {channel.name} directory for new videos')
- refresh_channel_videos(channel, reporter)
+ refresh_channel_videos(channel)
+
+ if not channel_links:
+ # Refresh NO CHANNEL videos when not refreshing a specific channel.
+ refresh_no_channel_videos()
# Fill in any missing data for all videos.
- process_video_meta_data()
-
- reporter.send_progress(0, len(channels), 'All videos refreshed.')
- reporter.code('refresh-complete')
+ if not PYTEST:
+ process_video_meta_data()
def get_channels_config(session: Session) -> dict:
@@ -319,43 +343,20 @@ async def get_statistics():
return ret
-async def get_download_history():
- """
- Get the info_date for all channels that have been downloaded. Order by the most recent download first.
- """
- query = '''
- SELECT
- link, name, info_date
- FROM
- channel
- WHERE
- info_date IS NOT NULL
- ORDER BY channel.info_date DESC
- '''
- with get_db_curs() as curs:
- curs.execute(query)
- results = [dict(i) for i in curs.fetchall()]
- return results
-
-
NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{11})_)?(.*)\.'
r'(jpg|webp|flv|mp4|part|info\.json|description|webm|..\.srt|..\.vtt)')
-def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel, idempotency: str = None,
- skip_captions=False,
- id_: str = None, info_json: dict = None) -> Video:
+def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = None, idempotency: str = None,
+ skip_captions=False, id_: str = None) -> Video:
"""
Insert a video into the DB. Also, find any meta-files near the video file and store them on the video row.
If id_ is provided, update that entry.
"""
- channel_dir = get_absolute_media_path(channel.directory)
- poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path, relative_to=channel_dir)
-
- # Video paths should be relative to the channel's directory
- if video_path.is_absolute():
- video_path = video_path.relative_to(channel_dir)
+ if not video_path.is_absolute():
+ raise ValueError(f'Video path is not absolute: {video_path}')
+ poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path)
name_match = NAME_PARSER.match(video_path.name)
_ = upload_date = source_id = title = ext = None
@@ -369,28 +370,33 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel, i
upload_date = None
duration = None
+ url = None
if info_json_path:
- path = (channel_dir / info_json_path).absolute()
try:
- with open(path) as fh:
+ with info_json_path.open('rt') as fh:
json_contents = json.load(fh)
- duration = json_contents['duration']
+ duration = json_contents.get('duration')
+ url = json_contents.get('webpage_url')
except json.decoder.JSONDecodeError:
- logger.warning(f'Failed to load JSON file to get duration: {path}')
+ logger.warning(f'Failed to load JSON file to get duration: {info_json_path}')
+
+ size = video_path.stat().st_size
video_dict = dict(
- channel_id=channel.id,
+ caption_path=str(caption_path) if caption_path else None,
+ channel_id=channel.id if channel else None,
description_path=str(description_path) if description_path else None,
+ duration=duration,
ext=ext,
+ idempotency=idempotency,
+ info_json_path=str(info_json_path) if info_json_path else None,
poster_path=str(poster_path) if poster_path else None,
+ size=size,
source_id=source_id,
title=title,
upload_date=upload_date,
+ url=url,
video_path=str(video_path),
- caption_path=str(caption_path) if caption_path else None,
- idempotency=idempotency,
- info_json_path=str(info_json_path) if info_json_path else None,
- duration=duration,
)
if id_:
@@ -410,8 +416,7 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel, i
return video
-def find_meta_files(path: pathlib.Path, relative_to=None) -> Tuple[
- pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]:
+def find_meta_files(path: pathlib.Path) -> Tuple[pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]:
"""
Find all files that share a file's full path, except their extensions. It is assumed that file with the
same name, but different extension is related to that file. A None will be yielded if the meta file doesn't exist.
@@ -429,11 +434,7 @@ def find_meta_files(path: pathlib.Path, relative_to=None) -> Tuple[
for meta_ext in meta_exts:
meta_path = path.with_suffix(meta_ext)
if meta_path.exists():
- if relative_to:
- yield meta_path.relative_to(relative_to)
- break
- else:
- yield meta_path
- break
+ yield meta_path
+ break
else:
yield None
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 5c9ac131..f71925f3 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -1,27 +1,20 @@
import json
-from datetime import timedelta, datetime
+from datetime import datetime
from pathlib import Path
-from typing import Optional
+from typing import Optional, List
-from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, Computed, types
+from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, Computed
from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm.collections import InstrumentedList
-from wrolpi.common import Base, tsvector, ModelHelper, PathColumn, get_absolute_media_path
-from wrolpi.dates import now, today, TZDateTime
+from wrolpi.common import Base, tsvector, ModelHelper, logger
+from wrolpi.dates import now, TZDateTime
from wrolpi.db import get_db_curs
-from wrolpi.errors import UnknownVideo, NoFrequency, UnknownFile, UnknownDirectory
+from wrolpi.downloader import Download
+from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory
+from wrolpi.media_path import MediaPathType, MediaPath
-
-class ChannelPath(types.TypeDecorator):
- impl = types.String
-
- def process_bind_param(self, value, dialect):
- return str(value) if value else None
-
- def process_result_value(self, value, dialect):
- if value:
- return Path(value)
+logger = logger.getChild(__name__)
class Video(ModelHelper, Base):
@@ -32,12 +25,12 @@ class Video(ModelHelper, Base):
idempotency = Column(String)
# File paths
- caption_path = Column(ChannelPath)
- description_path = Column(ChannelPath)
+ caption_path = Column(MediaPathType)
+ description_path = Column(MediaPathType)
ext = Column(String)
- info_json_path = Column(ChannelPath)
- poster_path = Column(ChannelPath)
- video_path = Column(ChannelPath)
+ info_json_path = Column(MediaPathType)
+ poster_path = Column(MediaPathType)
+ video_path = Column(MediaPathType)
caption = Column(String)
duration = Column(Integer)
@@ -49,12 +42,13 @@ class Video(ModelHelper, Base):
validated_poster = Column(Boolean, default=False)
viewed = Column(TZDateTime)
view_count = Column(Integer)
+ url = Column(String)
textsearch = Column(tsvector, Computed('''to_tsvector('english'::regconfig,
((COALESCE(title, ''::text) || ' '::text) ||
COALESCE(caption, ''::text)))'''))
def __repr__(self):
- return f''
def dict(self) -> dict:
@@ -83,13 +77,13 @@ class Video(ModelHelper, Base):
"""
Remove all files related to this video. Add it to it's Channel's skip list.
- Raises: UnknownFile if a the video has no files.
+ Raises: UnknownFile if the video has no files.
"""
from modules.videos.common import get_absolute_video_files
video_files = get_absolute_video_files(self)
for path in video_files:
try:
- path.unlink()
+ path.path.unlink()
except FileNotFoundError:
pass
@@ -117,13 +111,9 @@ class Video(ModelHelper, Base):
"""
If this Video has an info_json file, return it's contents. Otherwise, return None.
"""
- if not self.channel or not self.channel.directory:
- return
-
try:
if self.info_json_path:
- path = self.channel.get_relative_path(self.info_json_path)
- with open(path, 'rb') as fh:
+ with open(self.info_json_path.path, 'rb') as fh:
contents = json.load(fh)
return contents
except UnknownFile:
@@ -142,12 +132,10 @@ class Video(ModelHelper, Base):
if description:
return description
- if self.description_path and self.channel_id:
- path = self.channel.get_relative_path(self.description_path)
- if path.exists():
- with open(path, 'rt') as fh:
- contents = fh.read()
- return contents
+ if self.description_path:
+ with open(self.description_path, 'rt') as fh:
+ contents = fh.read()
+ return contents
def get_surrounding_videos(self):
"""
@@ -215,6 +203,28 @@ class Video(ModelHelper, Base):
return previous_video, next_video
+ def __json__(self):
+ from modules.videos.common import minimize_video_info_json
+ d = dict(
+ caption_path=self.caption_path,
+ channel=self.channel,
+ channel_id=self.channel_id,
+ duration=self.duration,
+ favorite=self.favorite,
+ id=self.id,
+ info_json=minimize_video_info_json(self.get_info_json()) if self.info_json_path else None,
+ poster_path=self.poster_path,
+ size=self.size,
+ source_id=self.source_id,
+ title=self.title,
+ upload_date=self.upload_date,
+ url=self.url,
+ video_path=self.video_path,
+ view_count=self.view_count,
+ viewed=self.viewed,
+ )
+ return d
+
class Channel(ModelHelper, Base):
__tablename__ = 'channel'
@@ -224,12 +234,12 @@ class Channel(ModelHelper, Base):
idempotency = Column(String)
url = Column(String)
match_regex = Column(String)
- directory = Column(PathColumn)
+ directory: MediaPath = Column(MediaPathType)
skip_download_videos = Column(ARRAY(String))
generate_posters = Column(Boolean)
calculate_duration = Column(Boolean)
download_frequency = Column(Integer)
- next_download = Column(Date)
+ source_id = Column(String)
info_json = Column(JSON)
info_date = Column(Date)
@@ -240,9 +250,9 @@ class Channel(ModelHelper, Base):
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
def __repr__(self):
- return f''
+ return f''
- def add_video_to_skip_list(self, source_id):
+ def add_video_to_skip_list(self, source_id: str):
if not source_id:
raise UnknownVideo(f'Cannot skip video with empty source id: {source_id}')
@@ -250,36 +260,6 @@ class Channel(ModelHelper, Base):
skip_download_videos.add(source_id)
self.skip_download_videos = skip_download_videos
- def increment_next_download(self):
- """
- Set the next download predictably during the next download iteration.
-
- For example, two channels that download weekly will need to be downloaded on different days. We want a channel
- to always be downloaded on it's day. That may be Monday, or Tuesday, etc.
-
- This is true for all download frequencies (30 days, 90 days, etc.).
-
- The order that channels will be downloaded/distributed will be by `link`.
- """
- if not self.download_frequency:
- raise NoFrequency(f'{self} does not have a frequency!')
-
- session = Session.object_session(self)
-
- # All the channels that share the my frequency.
- channel_group = session.query(self.__class__).filter_by(download_frequency=self.download_frequency)
- channel_group = list(channel_group.order_by(self.__class__.link).all())
-
- # My position in the channel group.
- index = channel_group.index(self)
-
- # The seconds between each download.
- chunk = self.download_frequency // len(channel_group)
-
- # My next download will be distributed by my frequency and my position.
- position = chunk * (index + 1)
- self.next_download = today() + timedelta(seconds=self.download_frequency + position)
-
def delete_with_videos(self):
"""
Delete all Video records (but not video files) related to this Channel. Then delete the Channel.
@@ -287,6 +267,9 @@ class Channel(ModelHelper, Base):
session = Session.object_session(self)
# Delete the video records, but not the video files!
session.query(Video).filter_by(channel_id=self.id).delete()
+ if download := self.get_download():
+ session.delete(download)
+
session.delete(self)
def update(self, data: dict):
@@ -298,35 +281,49 @@ class Channel(ModelHelper, Base):
Retrieve the data about this Channel that should be stored in a config file.
"""
config = dict(
- directory=str(self.directory),
+ calculate_duration=self.calculate_duration,
+ directory=str(self.directory.path),
+ download_frequency=self.download_frequency,
+ favorites={},
+ generate_posters=self.generate_posters,
match_regex=self.match_regex or '',
name=self.name,
- url=self.url or '',
- generate_posters=self.generate_posters,
- calculate_duration=self.calculate_duration,
skip_download_videos=self.skip_download_videos or [],
- download_frequency=self.download_frequency,
- next_download=self.next_download,
- favorites={},
+ source_id=self.source_id,
+ url=self.url or '',
)
session = Session.object_session(self)
- favorites = session.query(Video).filter(Video.favorite != None, Video.channel_id == self.id).all()
+ favorites = session.query(Video).filter(Video.favorite != None, Video.channel_id == self.id).all() # noqa
+ favorites: List[Video]
if favorites:
- config['favorites'] = {i.source_id: {'favorite': i.favorite} for i in favorites}
+ config['favorites'] = {i.video_path.path.name: {'favorite': i.favorite} for i in favorites}
return config
- def get_directory(self):
- if self._directory:
- return self._directory
-
- self._directory = get_absolute_media_path(self.directory)
- return self._directory
-
def get_relative_path(self, path: Path, exists: bool = True):
- channel_dir = self._directory or self.get_directory()
- path = channel_dir / path
+ path = self.directory / path
if exists and not path.exists():
- raise FileNotFoundError(f'Channel path {path} does not exist!')
+ raise FileNotFoundError(f'{path} does not exist!')
return path
+
+ def get_download(self) -> Optional[Download]:
+ """
+ Get the Download row for this Channel. If there isn't a Download, return None.
+ """
+ if not self.url:
+ raise ValueError(f'Channel {self.name} does not have a URL to download!')
+
+ session = Session.object_session(self)
+ download = session.query(Download).filter_by(url=self.url).one_or_none()
+ return download
+
+ def __json__(self):
+ d = dict(
+ id=self.id,
+ name=self.name,
+ directory=self.directory,
+ url=self.url,
+ link=self.link,
+ )
+ return d
diff --git a/modules/videos/schema.py b/modules/videos/schema.py
index ef727048..dd4ad698 100644
--- a/modules/videos/schema.py
+++ b/modules/videos/schema.py
@@ -11,7 +11,7 @@ class ChannelPostRequest:
mkdir = doc.Boolean()
generate_posters = doc.Boolean()
calculate_duration = doc.Boolean()
- download_frequency = doc.Integer()
+ download_frequency = doc.String()
class ChannelPutRequest:
@@ -23,7 +23,7 @@ class ChannelPutRequest:
generate_posters = doc.Boolean()
calculate_duration = doc.Boolean()
mkdir = doc.Boolean()
- download_frequency = doc.Integer()
+ download_frequency = doc.String()
class RefreshRequest:
@@ -170,13 +170,3 @@ class ChannelStatistics:
class VideosStatisticsResponse:
videos = VideoStatistics
channels = ChannelStatistics
-
-
-class DownloadHistory:
- link = doc.String()
- name = doc.String()
- info_date = doc.Date()
-
-
-class DownloadHistoryResponse:
- history = doc.List(doc.Object(DownloadHistory))
diff --git a/modules/videos/test/common.py b/modules/videos/test/common.py
index d13fc1f3..81f6af90 100644
--- a/modules/videos/test/common.py
+++ b/modules/videos/test/common.py
@@ -1,9 +1,10 @@
+import pathlib
+import tempfile
from functools import wraps
-from queue import Queue
from modules.videos.lib import refresh_channel_videos
from modules.videos.models import Channel
-from wrolpi.common import ProgressReporter, insert_parameter
+from wrolpi.common import insert_parameter, set_test_media_directory
from wrolpi.db import get_db_session
from wrolpi.test.common import wrap_test_db, build_test_directories
@@ -37,9 +38,11 @@ def create_channel_structure(structure):
@wraps(func)
@wrap_test_db
def wrapped(*args, **kwargs):
- # Dummy queue and reporter to receive messages.
- q = Queue()
- reporter = ProgressReporter(q, 2)
+ if args and (test := args[0]) and hasattr(test, 'tmp_dir'):
+ tmp_dir = pathlib.Path(test.tmp_dir.name)
+ else:
+ tmp_dir = pathlib.Path(tempfile.TemporaryDirectory().name)
+ set_test_media_directory(tmp_dir)
# Convert the channel/video structure to a file structure for the test.
file_structure = []
@@ -48,7 +51,7 @@ def create_channel_structure(structure):
file_structure.append(f'{channel}/{path}')
file_structure.append(f'{channel}/')
- with build_test_directories(file_structure) as tempdir:
+ with build_test_directories(file_structure, tmp_dir) as tempdir:
args, kwargs = insert_parameter(func, 'tempdir', tempdir, args, kwargs)
with get_db_session(commit=True) as session:
@@ -60,9 +63,12 @@ def create_channel_structure(structure):
with get_db_session(commit=True) as session:
for channel_ in session.query(Channel).all():
- refresh_channel_videos(channel_, reporter)
+ refresh_channel_videos(channel_)
- return func(*args, **kwargs)
+ try:
+ return func(*args, **kwargs)
+ finally:
+ set_test_media_directory(None)
return wrapped
diff --git a/modules/videos/test/test_captions.py b/modules/videos/test/test_captions.py
index a2c5e6d7..489082fe 100644
--- a/modules/videos/test/test_captions.py
+++ b/modules/videos/test/test_captions.py
@@ -1,15 +1,22 @@
-import mock
+from pathlib import Path
from modules.videos import captions
from modules.videos.models import Video
-from wrolpi.db import get_db_session, get_db_curs
+from wrolpi.db import get_db_curs, get_db_context
from wrolpi.test.common import wrap_test_db, TestAPI
from wrolpi.vars import PROJECT_DIR
class TestCaption(TestAPI):
- vtt_path1 = (PROJECT_DIR / 'test/example1.en.vtt').absolute()
- vtt_path2 = (PROJECT_DIR / 'test/example2.en.vtt').absolute()
+
+ def setUp(self) -> None:
+ super().setUp()
+ self.vtt_path1 = Path(self.tmp_dir.name) / 'example1.en.vtt'
+ self.vtt_path1.touch()
+ self.vtt_path1.write_text((PROJECT_DIR / 'test/example1.en.vtt').read_text())
+ self.vtt_path2 = Path(self.tmp_dir.name) / 'example2.en.vtt'
+ self.vtt_path2.touch()
+ self.vtt_path2.write_text((PROJECT_DIR / 'test/example2.en.vtt').read_text())
def test_example_en_vtt(self):
expected = ['okay welcome to this session this is',
@@ -35,25 +42,26 @@ class TestCaption(TestAPI):
@wrap_test_db
def test_process_captions(self):
- with get_db_session(commit=True) as session:
- video1 = Video(title='scream', caption_path=str(self.vtt_path1))
- session.add(video1)
- with mock.patch('modules.videos.captions.get_absolute_video_caption', lambda *a: self.vtt_path1):
- captions.process_captions(video1)
- video2 = Video(title='bar', caption_path=str(self.vtt_path2))
- session.add(video2)
- with mock.patch('modules.videos.captions.get_absolute_video_caption', lambda *a: self.vtt_path2):
- captions.process_captions(video2)
+ _, session = get_db_context()
+ video1 = Video(title='scream', caption_path=str(self.vtt_path1))
+ session.add(video1)
+ session.commit()
+ captions.process_captions(video1)
+ video2 = Video(title='bar', caption_path=str(self.vtt_path2))
+ session.add(video2)
+ session.commit()
+ captions.process_captions(video2)
- session.flush()
- session.refresh(video1)
- session.refresh(video2)
+ session.commit()
+ session.flush()
+ session.refresh(video1)
+ session.refresh(video2)
- # Get the video from the DB
- video1 = session.query(Video).filter_by(id=video1.id).one()
- self.assertIsNotNone(video1.caption)
- video2 = session.query(Video).filter_by(id=video2.id).one()
- self.assertIsNotNone(video2.caption)
+ # Get the video from the DB
+ video1 = session.query(Video).filter_by(id=video1.id).one()
+ self.assertIsNotNone(video1.caption)
+ video2 = session.query(Video).filter_by(id=video2.id).one()
+ self.assertIsNotNone(video2.caption)
# Search using the tsvector, "sessions" never actually appears in the text, but "session" does
with get_db_curs() as curs:
diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py
index aaa40a5e..8638b5b9 100644
--- a/modules/videos/test/test_common.py
+++ b/modules/videos/test/test_common.py
@@ -2,7 +2,7 @@ import pathlib
import subprocess
import tempfile
from unittest import mock
-from unittest.mock import Mock
+from unittest.mock import Mock, MagicMock
import pytest
from PIL import Image
@@ -11,17 +11,17 @@ from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.common import get_absolute_media_path
from wrolpi.db import get_db_session
-from wrolpi.test.common import ExtendedTestCase, build_test_directories, wrap_test_db
+from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
from wrolpi.vars import PROJECT_DIR
from ..common import get_matching_directories, convert_image, bulk_validate_posters, remove_duplicate_video_paths, \
update_view_count, get_video_duration
-class TestCommon(ExtendedTestCase):
+class TestCommon(TestAPI):
def test_get_absolute_media_path(self):
- wrolpi = get_absolute_media_path('videos/wrolpi')
- assert str(wrolpi).endswith('wrolpi')
+ path = get_absolute_media_path('videos')
+ self.assertTrue(str(path).endswith('videos'))
def test_matching_directories(self):
structure = [
@@ -59,13 +59,13 @@ class TestCommon(ExtendedTestCase):
"""
Video duration can be retrieved from the video file.
"""
- video_path = PROJECT_DIR / 'test/videos/wrolpi/big_buck_bunny_720p_1mb.mp4'
+ video_path = PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4'
self.assertEqual(get_video_duration(video_path), 5)
- video_path = PROJECT_DIR / 'test/videos/wrolpi/does not exist.mp4'
+ video_path = PROJECT_DIR / 'test/does not exist.mp4'
self.assertRaises(FileNotFoundError, get_video_duration, video_path)
- video_path = str(PROJECT_DIR / 'test/videos/wrolpi/does not exist.mp4')
+ video_path = str(PROJECT_DIR / 'test/does not exist.mp4')
self.assertRaises(FileNotFoundError, get_video_duration, video_path)
with tempfile.NamedTemporaryFile() as fh:
@@ -139,7 +139,7 @@ def test_convert_image():
)
def test_create_db_structure(_structure, paths):
@create_channel_structure(_structure)
- def test_func(tempdir):
+ def test_func(_, tempdir):
assert isinstance(tempdir, pathlib.Path)
for path in paths:
path = (tempdir / path)
@@ -154,7 +154,12 @@ def test_create_db_structure(_structure, paths):
assert channel.directory == tempdir / channel_name
assert len(channel.videos) == len([i for i in _structure[channel_name] if i.endswith('mp4')])
- test_func()
+ with tempfile.TemporaryDirectory() as tmp_dir, \
+ mock.patch('wrolpi.media_path.get_media_directory') as mock_get_directory:
+ mock_get_directory.return_value = pathlib.Path(tmp_dir)
+ test_ = MagicMock()
+ test_.tmp_dir.name = tmp_dir
+ test_func(test_)
@wrap_test_db
@@ -184,10 +189,10 @@ def test_bulk_replace_invalid_posters(tempdir: pathlib.Path):
assert Image.open(webp_fh).format == 'WEBP'
with get_db_session() as session:
- vid1 = session.query(Video).filter_by(poster_path='vid1.jpg').one()
+ vid1 = session.query(Video).filter_by(poster_path=f'{tempdir}/channel1/vid1.jpg').one()
assert vid1.validated_poster is False
- vid2 = session.query(Video).filter_by(poster_path='vid2.webp').one()
+ vid2 = session.query(Video).filter_by(poster_path=f'{tempdir}/channel2/vid2.webp').one()
assert vid2.validated_poster is False
# Convert the WEBP image. convert_image() should only be called once.
@@ -201,13 +206,13 @@ def test_bulk_replace_invalid_posters(tempdir: pathlib.Path):
with get_db_session() as session:
# Get the video by ID because it's poster is now a JPEG.
vid2 = session.query(Video).filter_by(id=vid2.id).one()
- assert str(vid2.poster_path) == 'vid2.jpg'
- assert all('webp' not in str(i.poster_path) for i in session.query(Video).all())
+ assert str(vid2.poster_path.path).split('/')[-1] == 'vid2.jpg'
+ assert all('webp' not in str(i.poster_path.path) for i in session.query(Video).all())
assert vid2.validated_poster is True
# Vid1's image was validated, but not converted.
vid1 = session.query(Video).filter_by(id=vid1.id).one()
- assert str(vid1.poster_path) == 'vid1.jpg'
+ assert str(vid1.poster_path.path).split('/')[-1] == 'vid1.jpg'
assert vid1.validated_poster is True
# Old webp was removed
@@ -240,9 +245,9 @@ def test_bulk_replace_invalid_posters(tempdir: pathlib.Path):
)
def test_update_view_count(tempdir: pathlib.Path):
def check_view_counts(view_counts):
- with get_db_session(commit=True) as session:
+ with get_db_session() as session_:
for source_id, view_count in view_counts.items():
- vid = session.query(Video).filter_by(source_id=source_id).one()
+ vid = session_.query(Video).filter_by(source_id=source_id).one()
assert vid.view_count == view_count
with get_db_session(commit=True) as session:
@@ -251,9 +256,9 @@ def test_update_view_count(tempdir: pathlib.Path):
channel2.info_json = {'entries': [{'id': 'vid2.mp4', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]}
channel3.info_json = {'entries': [{'id': 'vid3.mp4', 'view_count': 13}, {'id': 'vid4.mp4', 'view_count': 14}]}
- # Use the video_path as a unique source_id
+ # Use the video file name as a unique source_id
for v in session.query(Video).all():
- v.source_id = str(v.video_path)
+ v.source_id = str(v.video_path.path).split('/')[-1]
# Check all videos are empty.
check_view_counts({'vid1.mp4': None, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None})
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index 650ff214..a34023e8 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -1,17 +1,18 @@
-import time
-from datetime import timedelta
-from queue import Queue
+import pathlib
+from copy import copy
+from datetime import datetime
from unittest import mock
-from unittest.mock import MagicMock
-import pytest
+from yt_dlp.utils import UnsupportedError
-from modules.videos.downloader import update_channels, find_all_missing_videos, download_video
+from modules.videos.channel.lib import spread_channel_downloads
+from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
+ ChannelDownloader, get_or_create_channel
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
-from wrolpi.common import ProgressReporter
-from wrolpi.dates import today
-from wrolpi.db import get_db_session
+from wrolpi.dates import local_timezone
+from wrolpi.db import get_db_session, get_db_context
+from wrolpi.downloader import DownloadManager, Download, DownloadFrequency
from wrolpi.test.common import wrap_test_db, TestAPI
@@ -26,67 +27,13 @@ class FakeYDL:
}
-@wrap_test_db
-@create_channel_structure(
- {
- 'channel1': ['vid1.mp4'],
- 'channel2': ['vid2.mp4']
- },
-)
-def test_update_channels(tempdir):
- """
- Channels are only updated if their "next_download" has expired, or if they have never been updated.
- """
- q = Queue()
- reporter = ProgressReporter(q, 2)
-
- with get_db_session(commit=True) as session:
- channel1, channel2 = session.query(Channel).order_by(Channel.id).all()
- channel1.url = channel2.url = 'some url'
-
- with get_db_session() as session:
- channel1, channel2 = session.query(Channel).order_by(Channel.id).all()
- with mock.patch('modules.videos.downloader.update_channel') as update_channel:
- update_channel: MagicMock
- update_channels(reporter)
-
- # Both channels would have been updated.
- assert update_channel.call_count == 2
- update_channel.assert_any_call(channel1)
- update_channel.assert_any_call(channel2)
-
- with get_db_session() as session:
- channel1, channel2 = session.query(Channel).order_by(Channel.id).all()
- channel1.next_download = today() + timedelta(days=1)
- session.commit()
-
- with mock.patch('modules.videos.downloader.update_channel') as update_channel:
- update_channel: MagicMock
- update_channels(reporter)
-
- # Channel1 is not ready for an update
- update_channel.assert_called_once()
- update_channel.assert_called_with(channel2)
-
- channel1.next_download = today() + timedelta(days=1)
- channel2.next_download = today() + timedelta(days=1)
- session.commit()
-
- with mock.patch('modules.videos.downloader.update_channel') as update_channel:
- update_channel: MagicMock
- update_channels(reporter)
-
- # No channels needed to be updated
- assert update_channel.call_count == 0
-
-
class TestDownloader(TestAPI):
@wrap_test_db
@create_channel_structure(
{
'channel1': ['vid1.mp4'],
'channel2': ['vid2.mp4']
- },
+ }
)
def test_find_all_missing_videos(self, tempdir):
with get_db_session(commit=True) as session:
@@ -106,103 +53,310 @@ class TestDownloader(TestAPI):
self.assertEqual(len(missing), 1)
# The video returned is the one we faked.
- channel, id_, entry = missing[0]
- self.assertEqual(channel, channel1)
+ id_, source_id, entry = missing[0]
# Two videos were created for this test already.
self.assertEqual(id_, 3)
# The fake entry we added is regurgitated back.
self.assertEqual(entry, channel1.info_json['entries'][0])
- @pytest.mark.skip
- def test_download_timeout(self):
- class FakeYoutubeDL:
- def __init__(self, *a, **kw):
- pass
- def add_default_info_extractors(self):
- pass
+example_video_json = {
+ 'age_limit': 0,
+ 'average_rating': 4.6923075,
+ 'categories': ['Howto & Style'],
+ 'channel': 'channel name',
+ 'channel_id': 'channel id',
+ 'channel_url': 'channel url',
+ 'duration': 635,
+ 'extractor': 'youtube',
+ 'extractor_key': 'Youtube',
+ 'id': 'some long id',
+ 'is_live': None,
+ 'like_count': 24,
+ 'subtitles': {},
+ 'title': "The video's title",
+ 'upload_date': '20190707',
+ 'uploader': 'uploader name',
+ 'uploader_id': 'uploader id',
+ 'uploader_url': 'uploader url',
+ 'view_count': 406,
+ 'webpage_url': 'webpage url',
+ 'webpage_url_basename': 'watch',
+}
- def extract_info(self, *a, **kw):
- # Sleep longer than the 1 second timeout.
- for i in range(10):
- time.sleep(1)
+example_channel_json = {
+ '_type': 'playlist',
+ 'entries': [
+ {'_type': 'url',
+ 'description': None,
+ 'duration': None,
+ 'id': 'video 1 id',
+ 'ie_key': 'Youtube',
+ 'title': 'video 1 title',
+ 'uploader': None,
+ 'url': 'video 1 url',
+ 'view_count': 58504},
+ {'_type': 'url',
+ 'description': None,
+ 'duration': None,
+ 'id': 'video 2 id',
+ 'ie_key': 'Youtube',
+ 'title': 'video 2 title',
+ 'uploader': None,
+ 'url': 'video 2 url',
+ 'view_count': 1413},
+ ],
+ 'extractor': 'youtube:tab',
+ 'extractor_key': 'YoutubeTab',
+ 'id': 'some id',
+ 'title': 'channel title',
+ 'webpage_url': 'channel url',
+ 'webpage_url_basename': 'videos',
+}
- with mock.patch('api.timeout.TEST_TIMEOUT', 1), \
- mock.patch('modules.videos.downloader.YoutubeDL', FakeYoutubeDL), \
- mock.patch('modules.videos.downloader.get_absolute_media_path'):
- channel = Channel()
- self.assertRaises(TimeoutError, download_video, channel, {'id': 1})
- # {key: 'daily', text: 'Daily', value: 86400},
- # {key: 'weekly', text: 'Weekly', value: 604800},
- # {key: 'biweekly', text: 'Biweekly', value: 1209600},
- # {key: '30days', text: '30 Days', value: 2592000},
- # {key: '90days', text: '90 Days', value: 7776000},
+class TestVideosDownloaders(TestAPI):
+
+ def setUp(self) -> None:
+ super().setUp()
+ self.vd = VideoDownloader()
+ self.cd = ChannelDownloader()
+ self.mgr = DownloadManager()
+ self.mgr.register_downloader(self.vd)
+ self.mgr.register_downloader(self.cd)
+
+ self.videos_dir = pathlib.Path(self.tmp_dir.name) / 'videos'
+ self.videos_dir.mkdir()
+
+ def test_video_valid_url(self):
+ # A specific video can be downloaded.
+ self.assertTrue(self.vd.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c'))
+
+ # A channel cannot be downloaded.
+ self.assertFalse(self.vd.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos'))
+ self.assertFalse(self.vd.valid_url('https://www.youtube.com/c/LearningSelfReliance'))
+
+ def test_channel_valid_url(self):
+ # An entire domain cannot be downloaded.
+ self.assertFalse(self.cd.valid_url('https://example.com'))
+ self.assertFalse(self.cd.valid_url('https://youtube.com'))
+
+ # Cannot download a single video.
+ self.assertFalse(self.cd.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c'))
+
+ # Can download entire channels.
+ self.assertTrue(self.cd.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos'))
+ self.assertTrue(self.cd.valid_url('https://www.youtube.com/c/LearningSelfReliance'))
+
+ # Can download entire playlists.
+ self.assertTrue(self.cd.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E'))
+
+ def test_get_downloader(self):
+ # The correct Downloader is gotten.
+ self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance/videos'), self.cd)
+ self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance'), self.cd)
+ self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c'), self.vd)
+
+ @staticmethod
+ def _make_video_files(channel_dir):
+ video_path = channel_dir / 'video.mp4'
+ poster_path = channel_dir / 'video.jpg'
+ description_path = channel_dir / 'video.description'
+ info_json_path = channel_dir / 'video.info.json'
+ video_path.touch()
+ poster_path.touch()
+ description_path.touch()
+ info_json_path.touch()
+ return video_path, poster_path, description_path, info_json_path
@wrap_test_db
- @create_channel_structure(
- {
- 'channel1': [], # More channels than days in a week.
- 'channel2': [],
- 'channel3': [],
- 'channel4': [],
- 'channel5': [],
- 'channel6': [],
- 'channel7': [],
- 'channel8': [],
- 'channel9': [],
- },
- )
- @mock.patch('modules.videos.downloader.YDL', FakeYDL())
- def test_update_channel_order(self, tempdir):
- q = Queue()
- reporter = ProgressReporter(q, 2)
+ def test_video_download(self):
+ channel_dir = self.videos_dir / 'channel name'
+ channel_dir.mkdir(parents=True)
- # All channels are downloaded weekly.
+ # Video paths
+ video_path, poster_path, description_path, info_json_path = self._make_video_files(channel_dir)
+
+ with get_db_session() as session:
+ url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
+ with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
+ mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
+ mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
+ mock_extract_info.return_value = example_video_json
+ mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+ mock_process_runner.return_value = 0
+
+ self.mgr.create_download(url, session)
+
+ mock_process_runner.assert_called_once()
+ video_url, _, out_dir = mock_process_runner.call_args[0]
+
+ download: Download = session.query(Download).one()
+ self.assertEqual(video_url, download.url)
+ channel: Channel = session.query(Channel).one()
+ self.assertEqual(channel.directory, channel_dir)
+
+ video: Video = session.query(Video).one()
+ self.assertEqual(video.video_path, video_path)
+ self.assertEqual(video.description_path, description_path)
+ self.assertEqual(video.poster_path, poster_path)
+ self.assertEqual(video.info_json_path, info_json_path)
+ self.assertTrue(video.video_path.path.is_absolute())
+ self.assertTrue(video.poster_path.path.is_absolute())
+ self.assertTrue(video.description_path.path.is_absolute())
+ self.assertTrue(video.info_json_path.path.is_absolute())
+
+ @wrap_test_db
+ def test_video_download_no_channel(self):
+ _, session = get_db_context()
+
+ channel_dir = self.videos_dir / 'NO CHANNEL'
+ channel_dir.mkdir(parents=True)
+
+ # Video paths
+ video_path, _, _, _ = self._make_video_files(channel_dir)
+
+ # Video has no channel
+ info_json = copy(example_video_json)
+ del info_json['channel']
+ del info_json['channel_id']
+ del info_json['channel_url']
+
+ url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
+ with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
+ mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
+ mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
+ mock_extract_info.return_value = info_json
+ mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+ mock_process_runner.return_value = 0
+
+ self.mgr.create_download(url, session)
+
+ mock_process_runner.assert_called_once()
+
+ video: Video = session.query(Video).one()
+ self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4')
+
+ @wrap_test_db
+ def test_channel_download(self):
+ _, session = get_db_context()
+
+ url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
+ session.add(Channel(url=url, link='foo'))
+
+ channel_dir = self.videos_dir / 'NO CHANNEL'
+ channel_dir.mkdir(parents=True)
+ video_path, _, _, _ = self._make_video_files(channel_dir)
+
+ with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
+ mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
+ mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
+ mock_extract_info.return_value = example_channel_json
+ mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+ mock_process_runner.return_value = 0
+ self.mgr.create_download(url, session)
+
+ # Two videos are in the example channel.
+ downloads = self.mgr.get_new_downloads(session)
+ expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o']
+ for download, expected in zip(downloads, expected):
+ self.assertEqual(download.url, expected)
+ # Download is skipped for test.
+ self.assertEqual(download.status, 'new')
+
+ @wrap_test_db
+ def test_invalid_download_url(self):
+ _, session = get_db_context()
+ with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \
+ mock.patch('modules.videos.downloader.YDL') as mock_ydl:
+ mock_valid_url.return_value = True
+ mock_ydl.extract_info.side_effect = UnsupportedError('oops')
+ self.mgr.create_download('url', session)
+
+ @wrap_test_db
+ @mock.patch('modules.videos.channel.lib.today', lambda: local_timezone(datetime(2020, 1, 1, 0, 0, 0)))
+ def test_spread_channel_downloads(self):
with get_db_session(commit=True) as session:
- channels = list(session.query(Channel).order_by(Channel.link).all())
- for channel in channels:
- channel.download_frequency = 604800 # one week
- channel.url = 'some url'
- self.assertEqual(channel.next_download, None)
+ channel1 = Channel(name='channel1', link='channel1', url='https://example.com/1',
+ download_frequency=DownloadFrequency.weekly)
+ channel2 = Channel(name='channel2', link='channel2', url='https://example.com/2',
+ download_frequency=DownloadFrequency.weekly)
+ session.add_all([channel1, channel2])
- # Channels will download on different days the next week. Except where there are not enough days.
- update_channels(reporter)
- days = [7, 8, 9, 10, 10, 11, 12, 13, 14]
- for days_, channel in zip(days, channels):
- nd, expected = channel.next_download, today() + timedelta(days=days_)
- self.assertEqual(nd, expected, f'Expected {(nd - expected).days} days for {channel}')
+ def check_frequencies(expected):
+ with get_db_session(commit=True) as session:
+ downloads = list(session.query(Download).all())
+ self.assertLength(downloads, expected)
+ for download in downloads:
+ next_download = expected[download.url]
+ self.assertEqual(next_download, str(download.next_download),
+ f'{download.url} frequency {next_download} != {download.next_download}')
- # Even though all channels aren't updated, their order is the same.
- with get_db_session(commit=True):
- channels[0].next_download = None
- channels[1].next_download = None
- channels[4].next_download = None
- channels[8].next_download = None
- update_channels(reporter)
- for days_, channel in zip(days, channels):
- self.assertEqual(channel.next_download, today() + timedelta(days=days_))
+ # Spread the downloads out.
+ spread_channel_downloads()
+
+ expected = {
+ 'https://example.com/1': '2020-01-08 00:00:00-07:00',
+ 'https://example.com/2': '2020-01-11 12:00:00-07:00',
+ }
+ check_frequencies(expected)
- # Multiple frequencies are supported.
with get_db_session(commit=True) as session:
- frequency_map = {
- 'channel1': 604800, # weekly
- 'channel2': 604800,
- 'channel3': 1209600, # bi-weekly
- 'channel4': 1209600,
- 'channel5': 2592000, # 30 days
- 'channel6': 2592000,
- 'channel7': 2592000,
- 'channel8': 2592000,
- 'channel9': 2592000,
- }
- for channel in channels:
- channel.download_frequency = frequency_map[channel.link]
- channel.next_download = None
- update_channels(reporter)
- days = [10, 14, 21, 28,
- 36, 42, 48, 54, 60 # 6 days apart = 30 days / 5
- ]
- for days_, channel in zip(days, channels):
- nd, expected = channel.next_download, today() + timedelta(days=days_)
- self.assertEqual(nd, expected, f'Expected {(nd - expected).days} days for {channel}')
+ channel3 = Channel(name='channel3', link='channel3', url='https://example.com/3',
+ download_frequency=DownloadFrequency.daily)
+ channel4 = Channel(name='channel4', link='channel4', url='https://example.com/4',
+ download_frequency=DownloadFrequency.weekly)
+ # 5 and 6 should be ignored.
+ channel5 = Channel(name='channel5', link='channel5', url='https://example.com/5')
+ channel6 = Channel(name='channel6', link='channel6', download_frequency=DownloadFrequency.daily)
+ session.add_all([channel3, channel4, channel5, channel6])
+
+ spread_channel_downloads()
+ expected = {
+ 'https://example.com/1': '2020-01-08 00:00:00-07:00',
+ 'https://example.com/2': '2020-01-10 08:00:00-07:00',
+ 'https://example.com/3': '2020-01-02 00:00:00-07:00',
+ 'https://example.com/4': '2020-01-12 16:00:00-07:00',
+ }
+ check_frequencies(expected)
+
+ @wrap_test_db
+ def test_get_or_create_channel(self):
+ """
+ A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can
+ match it.
+ """
+ with get_db_session(commit=True) as session:
+ c1 = Channel(name='foo', link='foo', source_id='foo', url='foo')
+ c2 = Channel(name='bar', link='bar', source_id='bar')
+ c3 = Channel(name='baz', link='baz', source_id='baz', url='baz')
+ c4 = Channel(name='qux', link='qux')
+ session.add_all([c1, c2, c3, c4])
+
+ # All existing channels should be used.
+ tests = [
+ (dict(source_id='foo'), c1),
+ (dict(link='foo'), c1),
+ (dict(url='foo'), c1),
+ (dict(url='foo', source_id='bar'), c2), # source_id is preferred.
+ (dict(name='foo', source_id='bar'), c2),
+ (dict(source_id='bar'), c2),
+ (dict(source_id='baz'), c3),
+ (dict(name='qux'), c4),
+ (dict(link='qux'), c4),
+ ]
+ for kwargs, expected in tests:
+ channel = get_or_create_channel(**kwargs)
+ self.assertEqual(expected.id, channel.id, f'Expected {expected} got {channel}')
+
+ # A new channel is created. It will not be automatically downloaded.
+ channel = get_or_create_channel(source_id='quux', name='quux', url='quux')
+ self.assertEqual(channel.id, 5)
+ self.assertEqual(channel.source_id, 'quux')
+ self.assertEqual(channel.name, 'quux')
+ self.assertEqual(channel.link, 'quux')
+ self.assertEqual(channel.url, 'quux')
+ self.assertIsNone(channel.download_frequency)
+
+ # New channel can be retrieved.
+ self.assertEqual(get_or_create_channel(source_id='quux'), channel)
diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py
new file mode 100644
index 00000000..46b11c3b
--- /dev/null
+++ b/modules/videos/test/test_lib.py
@@ -0,0 +1,74 @@
+from datetime import datetime
+from unittest import mock
+
+import pytest
+
+from modules.videos.channel import lib
+from modules.videos.models import Channel
+from wrolpi.dates import local_timezone
+from wrolpi.downloader import DownloadFrequency
+
+daily, weekly, days30 = DownloadFrequency.daily, DownloadFrequency.weekly, DownloadFrequency.days30
+
+
+@pytest.mark.parametrize(
+ 'channels,expected',
+ [
+ ([Channel(download_frequency=daily, url='https://example.com/1')],
+ [('https://example.com/1', daily, '2000-01-02 00:00:00-07:00')]
+ ),
+ # Several different download frequencies.
+ ([
+ Channel(download_frequency=daily, url='https://example.com/1'),
+ Channel(download_frequency=daily, url='https://example.com/2'),
+ Channel(download_frequency=daily, url='https://example.com/3'),
+ Channel(download_frequency=weekly, url='https://example.com/4'),
+ Channel(download_frequency=weekly, url='https://example.com/5'),
+ Channel(download_frequency=days30, url='https://example.com/6'),
+ ],
+ [
+ ('https://example.com/1', daily, '2000-01-02 00:00:00-07:00'),
+ ('https://example.com/2', daily, '2000-01-02 08:00:00-07:00'),
+ ('https://example.com/3', daily, '2000-01-02 16:00:00-07:00'),
+ ('https://example.com/4', weekly, '2000-01-08 00:00:00-07:00'),
+ ('https://example.com/5', weekly, '2000-01-11 12:00:00-07:00'),
+ ('https://example.com/6', days30, '2000-01-31 00:00:00-07:00'),
+ ]
+ ),
+ # Multiple downloads in a day.
+ ([Channel(download_frequency=daily, url=f'https://example.com/{i}') for i in range(1, 3)],
+ [
+ ('https://example.com/1', daily, '2000-01-02 00:00:00-07:00'),
+ ('https://example.com/2', daily, '2000-01-02 12:00:00-07:00'),
+ ]
+ ),
+ # More downloads in a week than days in a week.
+ ([Channel(download_frequency=weekly, url=f'https://example.com/{i}') for i in range(1, 9)],
+ [
+ ('https://example.com/1', weekly, '2000-01-08 00:00:00-07:00'),
+ ('https://example.com/2', weekly, '2000-01-08 21:00:00-07:00'),
+ ('https://example.com/3', weekly, '2000-01-09 18:00:00-07:00'),
+ ('https://example.com/4', weekly, '2000-01-10 15:00:00-07:00'),
+ ('https://example.com/5', weekly, '2000-01-11 12:00:00-07:00'),
+ ('https://example.com/6', weekly, '2000-01-12 09:00:00-07:00'),
+ ('https://example.com/7', weekly, '2000-01-13 06:00:00-07:00'),
+ ('https://example.com/8', weekly, '2000-01-14 03:00:00-07:00'),
+ ]
+ ),
+ # Any Channel without a URL AND frequency is ignored.
+ ([
+ Channel(download_frequency=daily, url='https://example.com/1'),
+ Channel(download_frequency=daily),
+ Channel(url='this should be ignored'),
+ ],
+ [('https://example.com/1', daily, '2000-01-02 00:00:00-07:00')]
+ ),
+ ]
+)
+@mock.patch('modules.videos.channel.lib.today', lambda: local_timezone(datetime(2000, 1, 1, 0, 0, 0)))
+def test_spread_by_frequency(channels, expected):
+ result = lib._spread_by_frequency(channels)
+ for result, (expected_url, expected_frequency, expected_dt) in zip(result, expected):
+ assert result['url'] == expected_url
+ assert result['frequency'] == expected_frequency
+ assert str(result['next_download']) == expected_dt
diff --git a/modules/videos/test/test_video_url_resolver.py b/modules/videos/test/test_video_url_resolver.py
new file mode 100644
index 00000000..d9b67777
--- /dev/null
+++ b/modules/videos/test/test_video_url_resolver.py
@@ -0,0 +1,15 @@
+import pytest
+
+from modules.videos.video_url_resolver import video_url_resolver
+
+
+@pytest.mark.parametrize(
+ 'domain,entry,expected',
+ [
+ ('youtube.com', {'url': 'foo'}, 'https://youtube.com/watch?v=foo'),
+ ('youtube.com', {'webpage_url': 'https://youtube.com/watch?v=foo'}, 'https://youtube.com/watch?v=foo'),
+ ('youtube.com', {'url': 'https://youtube.com/watch?v=foo'}, 'https://youtube.com/watch?v=foo'),
+ ]
+)
+def test_video_url_resolver(domain, entry, expected):
+ assert video_url_resolver(domain, entry) == expected
diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py
index a4537458..0ddaa8e4 100644
--- a/modules/videos/video/api.py
+++ b/modules/videos/video/api.py
@@ -5,12 +5,12 @@ from sanic.request import Request
from wrolpi.common import logger, wrol_mode_check, run_after, get_media_directory, \
get_relative_to_media_directory
-from wrolpi.root_api import json_response
from wrolpi.db import get_db_session
from wrolpi.errors import ValidationError, InvalidOrderBy
+from wrolpi.root_api import json_response
from wrolpi.schema import validate_doc, JSONErrorResponse
from .lib import get_video, VIDEO_ORDERS, DEFAULT_VIDEO_ORDER, video_search, get_video_for_app
-from ..common import get_matching_directories, get_allowed_limit, minimize_video
+from ..common import get_matching_directories, get_allowed_limit
from ..lib import save_channels_config
from ..schema import VideoResponse, VideoSearchRequest, VideoSearchResponse, \
DirectoriesResponse, DirectoriesRequest
@@ -55,7 +55,7 @@ async def search(_: Request, data: dict):
videos, videos_total = video_search(search_str, offset, limit, channel_link, order_by, favorites)
- ret = {'videos': list(map(minimize_video, videos)), 'totals': {'videos': videos_total}}
+ ret = {'videos': list(videos), 'totals': {'videos': videos_total}}
return json_response(ret)
diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py
index e62dfaed..be33f579 100644
--- a/modules/videos/video/lib.py
+++ b/modules/videos/video/lib.py
@@ -31,9 +31,9 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
video.set_viewed()
previous_video, next_video = video.get_surrounding_videos()
- video = video.get_minimize()
- previous_video = previous_video.get_minimize() if previous_video else None
- next_video = next_video.get_minimize() if next_video else None
+ video = video.__json__()
+ previous_video = previous_video.__json__() if previous_video else None
+ next_video = next_video.__json__() if next_video else None
return video, previous_video, next_video
@@ -139,7 +139,7 @@ def video_search(
results = session.query(Video).filter(Video.id.in_(ranked_ids)).all()
results = sorted(results, key=lambda r: ranked_ids.index(r.id))
- results = [i.get_minimize() for i in results]
+ results = [i.__json__() for i in results]
return results, total
diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py
index fe45fff0..a4b2ed99 100644
--- a/modules/videos/video/test/test_api.py
+++ b/modules/videos/video/test/test_api.py
@@ -3,7 +3,6 @@ from datetime import timedelta
from http import HTTPStatus
from json import dumps
from unittest import mock
-from unittest.mock import MagicMock
from uuid import uuid4
from modules.videos.models import Channel, Video
@@ -102,13 +101,13 @@ class TestVideoFunctions(TestAPI):
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
# Can't refresh or download
- _, resp = api_app.test_client.post('/api/videos:refresh')
+ _, resp = api_app.test_client.post('/api/videos/refresh')
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
- _, resp = api_app.test_client.post('/api/videos:download')
+ _, resp = api_app.test_client.post('/api/videos/download')
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
# THE REST OF THESE METHODS ARE ALLOWED
- _, resp = api_app.test_client.post('/api/videos:favorite', data=favorite)
+ _, resp = api_app.test_client.post('/api/videos/favorite', data=favorite)
self.assertEqual(resp.status_code, HTTPStatus.OK)
@wrap_test_db
@@ -164,33 +163,3 @@ class TestVideoFunctions(TestAPI):
self.assertEqual(session.query(Video).count(), 2)
self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file())
self.assertFalse((tempdir / 'channel2/vid2.mp4').is_file())
-
- def test_events(self):
- request, response = api_app.test_client.get('/api/events')
- self.assertOK(response)
- self.assertGreater(len(response.json['events']), 1)
- self.assertFalse(any(i['is_set'] for i in response.json['events']))
-
- calls = []
-
- async def fake_refresh_videos(*a, **kw):
- calls.append((a, kw))
-
- with mock.patch('modules.videos.api.refresh_videos', fake_refresh_videos), \
- mock.patch('modules.videos.api.refresh_event') as refresh_event:
- refresh_event: MagicMock
-
- # Cannot start a second refresh while one is running.
- refresh_event.is_set.return_value = True
- request, response = api_app.test_client.post('/api/videos:refresh')
- self.assertCONFLICT(response)
-
- # Refresh is started, a stream is created
- refresh_event.is_set.return_value = False
- request, response = api_app.test_client.post('/api/videos:refresh')
- self.assertOK(response)
- self.assertEqual(response.json['code'], 'stream-started')
- stream_url: str = response.json['stream_url']
- assert stream_url.startswith('ws://')
- assert calls == [((None,), {})]
- refresh_event.set.assert_called()
diff --git a/modules/videos/video_url_resolver.py b/modules/videos/video_url_resolver.py
new file mode 100644
index 00000000..4bbdce0e
--- /dev/null
+++ b/modules/videos/video_url_resolver.py
@@ -0,0 +1,24 @@
+formatters = {
+ 'youtube.com': lambda i: f'https://youtube.com/watch?v={i["url"]}',
+ 'www.youtube.com': lambda i: f'https://youtube.com/watch?v={i["url"]}',
+}
+
+
+def video_url_resolver(domain: str, entry: dict) -> str:
+ """
+ Guess the full URL of a Youtube-DL entry.
+ """
+ if 'webpage_url' in entry:
+ return entry['webpage_url']
+
+ if entry.get('url', '').startswith('https://'):
+ return entry['url']
+ if entry.get('url', '').startswith('http://'):
+ return entry['url']
+
+ try:
+ formatter = formatters[domain]
+ return formatter(entry)
+ except KeyError:
+ # Unable to resolve, try the URL as is.
+ return entry['url']
diff --git a/requirements.txt b/requirements.txt
index 04c69c8a..8b983ca4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,11 +8,12 @@ psycopg2-binary==2.8.6
pytz==2021.1
PyYAML==5.4
requests==2.26.0
-Sanic-Cors==0.10.0.post3
+Sanic-Cors==1.0.1
sanic-openapi==0.6.2
-sanic==20.12.0
+sanic-testing==0.7.0
+sanic==21.9.1
SQLAlchemy==1.3.22
srt==3.4.1
-websockets==8.1
+websockets==10.1
webvtt-py==0.4.6
-youtube-dl==2021.6.6
+yt-dlp==2021.12.1
diff --git a/test/videos/wrolpi/big_buck_bunny_720p_1mb.mp4 b/test/big_buck_bunny_720p_1mb.mp4
similarity index 100%
rename from test/videos/wrolpi/big_buck_bunny_720p_1mb.mp4
rename to test/big_buck_bunny_720p_1mb.mp4
diff --git a/wrolpi/__init__.py b/wrolpi/__init__.py
index 56d15213..7fe8a8d8 100644
--- a/wrolpi/__init__.py
+++ b/wrolpi/__init__.py
@@ -1,9 +1,48 @@
+import inspect
+import multiprocessing
+
BEFORE_STARTUP_FUNCTIONS = []
def before_startup(func: callable):
"""
- Run a callable before startup of the WROLPi API.
+ Run a callable before startup of the WROLPi API. This will be called (and blocked on) once.
"""
BEFORE_STARTUP_FUNCTIONS.append(func)
return func
+
+
+def after_startup(func: callable):
+ """
+ Run a function after the startup of the WROLPi Sanic API. This will be run for each process!
+ """
+ from .root_api import api_app
+ api_app.after_server_start(func)
+ return func
+
+
+def limit_concurrent(limit: int):
+ """
+ Wrapper that limits the amount of concurrently running functions.
+ """
+ sema = multiprocessing.Semaphore(value=limit)
+
+ def wrapper(func: callable):
+ if inspect.iscoroutinefunction(func):
+ async def wrapped(*a, **kw):
+ acquired = sema.acquire(block=False)
+ if not acquired:
+ return
+ return await func(*a, **kw)
+
+ return wrapped
+ else:
+ def wrapped(*a, **kw):
+ acquired = sema.acquire(block=False)
+ if not acquired:
+ return
+ return func(*a, **kw)
+
+ return wrapped
+
+ return wrapper
diff --git a/wrolpi/common.py b/wrolpi/common.py
index 3413de48..e4d291b2 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -3,19 +3,17 @@ import collections
import inspect
import json
import logging
-import os
import pathlib
import queue
import re
import string
-from copy import deepcopy
from datetime import datetime, date
from functools import wraps
from itertools import islice
from multiprocessing import Event, Queue
from pathlib import Path
from typing import Union, Callable, Tuple, Dict, Mapping, List, Iterable
-from urllib.parse import urlunsplit
+from urllib.parse import urlunsplit, urlparse
import yaml
from cachetools import cached, TTLCache
@@ -25,11 +23,10 @@ from sqlalchemy import types
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
-from wrolpi.errors import WROLModeEnabled, UnknownDirectory
-from wrolpi.vars import CONFIG_PATH, EXAMPLE_CONFIG_PATH, PUBLIC_HOST, PUBLIC_PORT, LAST_MODIFIED_DATE_FORMAT, \
- PROJECT_DIR
+from wrolpi.errors import WROLModeEnabled
+from wrolpi.vars import CONFIG_PATH, EXAMPLE_CONFIG_PATH, PUBLIC_HOST, PUBLIC_PORT, PROJECT_DIR, PYTEST, MODULES_DIR
-logger = logging.getLogger(__name__)
+logger = logging.getLogger()
ch = logging.StreamHandler()
formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s')
ch.setFormatter(formatter)
@@ -165,107 +162,6 @@ def get_sanic_url(scheme: str = 'http', path: str = None, query: list = None, fr
return unparsed
-def make_progress_calculator(total):
- """
- Create a function that calculates the percentage of completion.
- """
-
- def progress_calculator(current) -> int:
- if current >= total:
- # Progress doesn't make sense, just return 100
- return 100
- percent = int((current / total) * 100)
- return percent
-
- return progress_calculator
-
-
-class ProgressReporter:
- """
- I am used to consistently send messages and progress(s) to a Websocket Feed.
- """
-
- def __init__(self, q: Queue, progress_count: int = 1):
- self.queue: Queue = q
- self.progresses = [{'percent': 0, 'total': 0, 'value': 0} for _ in range(progress_count)]
- self.calculators = [lambda _: None for _ in range(progress_count)]
-
- def _update(self, idx: int, **kwargs):
- if 'message' in kwargs and kwargs['message'] is None:
- # Message can't be cleared.
- kwargs.pop('message')
- self.progresses[idx].update(kwargs)
-
- def _send(self, code: str = None):
- msg = dict(
- progresses=deepcopy(self.progresses)
- )
- if code:
- msg['code'] = code
- self.queue.put(msg)
-
- def message(self, idx: int, msg: str, code: str = None):
- self._update(idx, message=msg)
- self._send(code)
-
- def code(self, code: str):
- self._send(code)
-
- def error(self, idx: int, msg: str = None):
- self.message(idx, msg, 'error')
-
- def set_progress_total(self, idx: int, total: int):
- self.progresses[idx]['total'] = total
- self.calculators[idx] = make_progress_calculator(total)
-
- def send_progress(self, idx: int, value: int, msg: str = None):
- kwargs = dict(value=value, percent=self.calculators[idx](value), message=msg)
- self._update(idx, **kwargs)
- self._send()
-
- def finish(self, idx: int, msg: str = None):
- kwargs = dict(percent=100, message=msg)
-
- if self.progresses[idx]['total'] == 0:
- kwargs.update(dict(value=1, total=1))
- else:
- kwargs.update(dict(value=self.progresses[idx]['total']))
-
- self._update(idx, **kwargs)
- self._send()
-
-
-class FileNotModified(Exception):
- pass
-
-
-def get_modified_time(path: Union[Path, str]) -> datetime:
- """
- Return a datetime object containing the os modification time of the provided path.
- """
- modified = datetime.utcfromtimestamp(os.path.getmtime(str(path)))
- return modified
-
-
-def get_last_modified_headers(request_headers: dict, path: Union[Path, str]) -> dict:
- """
- Get a dict containing the Last-Modified header for the provided path. If If-Modified-Since is in the provided
- request headers, then this will raise a FileNotModified exception, which should be handled by
- `handle_FileNotModified`.
- """
- last_modified = get_modified_time(path)
-
- modified_since = request_headers.get('If-Modified-Since')
- if modified_since:
- modified_since = datetime.strptime(modified_since, LAST_MODIFIED_DATE_FORMAT)
- if last_modified >= modified_since:
- raise FileNotModified()
-
- last_modified = last_modified.strftime(LAST_MODIFIED_DATE_FORMAT)
- headers = {'Last-Modified': last_modified}
- return headers
-
-
def get_example_config() -> dict:
config_path = EXAMPLE_CONFIG_PATH
with open(str(config_path), 'rt') as fh:
@@ -442,6 +338,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
return synchronous_after(*a, **kw)
def wrapper(func: callable):
+ if PYTEST:
+ return func
+
if inspect.iscoroutinefunction(func):
@wraps(func)
async def wrapped(*a, **kw):
@@ -468,7 +367,12 @@ MEDIA_DIRECTORY = None
def set_test_media_directory(path):
global TEST_MEDIA_DIRECTORY
- TEST_MEDIA_DIRECTORY = pathlib.Path(path) if path else None
+ if isinstance(path, pathlib.Path):
+ TEST_MEDIA_DIRECTORY = path
+ elif path:
+ TEST_MEDIA_DIRECTORY = pathlib.Path(path)
+ else:
+ TEST_MEDIA_DIRECTORY = None
def get_media_directory() -> Path:
@@ -478,6 +382,9 @@ def get_media_directory() -> Path:
global TEST_MEDIA_DIRECTORY
global MEDIA_DIRECTORY
+ if PYTEST and not TEST_MEDIA_DIRECTORY:
+ raise ValueError('No test media directory set during testing!!')
+
if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path):
return TEST_MEDIA_DIRECTORY
@@ -505,12 +412,9 @@ def get_absolute_media_path(path: str) -> Path:
:raises UnknownDirectory: the directory/path doesn't exist
"""
- media_directory = get_media_directory()
if not path:
- raise ValueError(f'Cannot combine empty path with {media_directory}')
- path = media_directory / path
- if not path.exists():
- raise UnknownDirectory(f'path={path}')
+ raise ValueError('Path cannot be empty!')
+ path = get_media_directory() / path
return path
@@ -543,3 +447,35 @@ def make_media_directory(path: str):
media_dir = get_media_directory()
path = media_dir / str(path)
path.mkdir(parents=True)
+
+
+def extract_domain(url):
+ """
+ Extract the domain from a URL. Remove leading www.
+
+ >>> extract_domain('https://www.example.com/foo')
+ 'example.com'
+ """
+ parsed = urlparse(url)
+ domain = parsed.netloc
+ domain = domain.decode() if hasattr(domain, 'decode') else domain
+ if domain.startswith('www.'):
+ # Remove leading www.
+ domain = domain[4:]
+ return domain
+
+
+def import_modules():
+ """
+ Import all WROLPi Modules in the modules directory. Raise an ImportError if there are no modules.
+ """
+ try:
+ modules = [i.name for i in MODULES_DIR.iterdir() if i.is_dir() and not i.name.startswith('_')]
+ for module in modules:
+ module = f'modules.{module}.api'
+ logger.debug(f'Importing {module}')
+ __import__(module, globals(), locals(), [], 0)
+ except ImportError as e:
+ logger.fatal('No modules could be found!', exc_info=e)
+ raise
+ return modules
diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py
index e96dc749..c1ad8c5e 100644
--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@@ -1,28 +1,66 @@
"""
Fixtures for Pytest tests.
"""
+import os
+import pathlib
+import tempfile
+from typing import Tuple
from unittest import mock
+from uuid import uuid1
import pytest
-from sqlalchemy.orm import Session
+import yaml
+from sanic_testing.testing import SanicTestClient
+from sqlalchemy.engine import Engine, create_engine
+from sqlalchemy.orm import Session, sessionmaker
-from wrolpi.db import postgres_engine
-from wrolpi.test.common import test_db
+from wrolpi.common import set_test_media_directory, Base, get_example_config
+from wrolpi.db import postgres_engine, get_db_args
+from wrolpi.root_api import BLUEPRINTS, api_app
+
+os.environ['DB_PORT'] = '54321'
-@pytest.fixture
+def get_test_db_engine():
+ suffix = str(uuid1()).replace('-', '')
+ db_name = f'wrolpi_testing_{suffix}'
+ conn = postgres_engine.connect()
+ conn.execute(f'DROP DATABASE IF EXISTS {db_name}')
+ conn.execute(f'CREATE DATABASE {db_name}')
+ conn.execute('commit')
+
+ test_args = get_db_args(db_name)
+ test_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**test_args))
+ return test_engine
+
+
+def test_db() -> Tuple[Engine, Session]:
+ """
+ Create a unique SQLAlchemy engine/session for a test.
+ """
+ test_engine = get_test_db_engine()
+ if test_engine.engine.url.database == 'wrolpi':
+ raise ValueError('Refusing the test on wrolpi database!')
+
+ # Create all tables. No need to check if they exist because this is a test DB.
+ Base.metadata.create_all(test_engine, checkfirst=False)
+ session = sessionmaker(bind=test_engine)()
+ return test_engine, session
+
+
+@pytest.fixture()
def test_session() -> Session:
"""
Pytest Fixture to get a test database session.
"""
test_engine, session = test_db()
- def fake_get_db_context():
+ def fake_get_db_session():
"""Get the testing db"""
return test_engine, session
try:
- with mock.patch('wrolpi.db._get_db_session', fake_get_db_context):
+ with mock.patch('wrolpi.db._get_db_session', fake_get_db_session):
yield session
finally:
session.rollback()
@@ -30,3 +68,45 @@ def test_session() -> Session:
test_engine.dispose()
conn = postgres_engine.connect()
conn.execute(f'DROP DATABASE IF EXISTS {test_engine.engine.url.database}')
+
+
+@pytest.fixture(autouse=True)
+def test_directory():
+ """
+ Overwrite the media directory with a temporary directory.
+ """
+ tmp_dir = tempfile.TemporaryDirectory()
+ tmp_path = pathlib.Path(tmp_dir.name)
+ set_test_media_directory(tmp_path)
+ return tmp_path
+
+
+@pytest.fixture(autouse=True)
+def test_config():
+ """
+ Create a test config based off the example config.
+ """
+ test_config_path = tempfile.NamedTemporaryFile(mode='rt')
+ with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name):
+ config = get_example_config()
+ with open(test_config_path.name, 'wt') as fh:
+ fh.write(yaml.dump(config))
+ yield
+
+
+ROUTES_ATTACHED = False
+
+
+@pytest.fixture(autouse=True)
+def test_client() -> SanicTestClient:
+ """
+ Get a Sanic Test Client with all default routes attached.
+ """
+ global ROUTES_ATTACHED
+ if ROUTES_ATTACHED is False:
+ # Attach any blueprints for the test.
+ for bp in BLUEPRINTS:
+ api_app.blueprint(bp)
+ ROUTES_ATTACHED = True
+
+ return api_app.test_client
diff --git a/wrolpi/dates.py b/wrolpi/dates.py
index 80662df4..f36871fd 100644
--- a/wrolpi/dates.py
+++ b/wrolpi/dates.py
@@ -85,9 +85,12 @@ class TZDateTime(types.TypeDecorator):
value = value.astimezone(timezone.utc)
elif isinstance(value, str) and '-' in value:
value = datetime.fromisoformat(value)
+ value = value.astimezone(timezone.utc)
return value
- def process_result_value(self, value, dialect):
+ def process_result_value(self, value: datetime, dialect):
if value is not None:
+ # Assume the DB timestamp is UTC if not specified.
+ value = value.replace(tzinfo=pytz.utc) if not value.tzinfo else value
value = local_timezone(value)
return value
diff --git a/wrolpi/db.py b/wrolpi/db.py
index 86111858..9123dcdc 100644
--- a/wrolpi/db.py
+++ b/wrolpi/db.py
@@ -1,5 +1,5 @@
from contextlib import contextmanager
-from typing import ContextManager
+from typing import ContextManager, Tuple
import psycopg2
import sqlalchemy.exc
@@ -51,7 +51,9 @@ LOGGED_ARGS = False
def _get_db_session():
- """This function allows the database to be wrapped during testing. See: api.test.common.wrap_test_db"""
+ """
+ This function allows the database to be wrapped during testing. See: api.test.common.wrap_test_db
+ """
global LOGGED_ARGS
if LOGGED_ARGS is False:
# Print the DB args for troubleshooting.
@@ -64,10 +66,19 @@ def _get_db_session():
return engine, session
+def get_db_context() -> Tuple[create_engine, Session]:
+ """
+ Get a DB engine and session.
+ """
+ return _get_db_session()
+
+
@contextmanager
def get_db_session(commit: bool = False) -> ContextManager[Session]:
- """Context manager that creates a DB session. This will automatically rollback changes, unless `commit` is True."""
- local_engine, session = _get_db_session()
+ """
+ Context manager that creates a DB session. This will automatically rollback changes, unless `commit` is True.
+ """
+ _, session = get_db_context()
try:
yield session
if commit:
@@ -83,8 +94,10 @@ def get_db_session(commit: bool = False) -> ContextManager[Session]:
@contextmanager
def get_db_curs(commit: bool = False):
- """Context manager that yields a DictCursor to execute raw SQL statements."""
- local_engine, session = _get_db_session()
+ """
+ Context manager that yields a DictCursor to execute raw SQL statements.
+ """
+ local_engine, session = get_db_context()
connection = local_engine.raw_connection()
curs = connection.cursor(cursor_factory=psycopg2.extras.DictCursor)
try:
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
new file mode 100644
index 00000000..3fa8070e
--- /dev/null
+++ b/wrolpi/downloader.py
@@ -0,0 +1,527 @@
+import asyncio
+import multiprocessing
+import pathlib
+import subprocess
+from collections import defaultdict
+from datetime import timedelta
+from enum import Enum
+from functools import partial
+from operator import attrgetter
+from typing import List, Optional
+
+from cachetools import cached, TTLCache
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.orm import Session
+
+from wrolpi.common import Base, ModelHelper, logger, iterify
+from wrolpi.dates import TZDateTime, now
+from wrolpi.db import get_db_session, get_db_curs, get_db_context
+from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
+from wrolpi.vars import PYTEST
+
+logger = logger.getChild(__name__)
+
+DOWNLOAD_IN_PROGRESS = multiprocessing.Semaphore()
+DEFAULT_RETRY_FREQUENCY = timedelta(hours=1)
+
+
+class DownloadFrequency(int, Enum):
+ hourly = 3600
+ daily = 86400
+ weekly = 604800
+ biweekly = 1209600
+ days30 = 2592000
+ days90 = 7776000
+
+
+class Download(ModelHelper, Base):
+ """
+ Model that is used to schedule downloads.
+ """
+ __tablename__ = 'download'
+ id = Column(Integer, primary_key=True)
+ url = Column(String, nullable=False)
+
+ attempts = Column(Integer, default=0)
+ frequency = Column(Integer)
+ last_successful_download = Column(TZDateTime)
+ next_download = Column(TZDateTime)
+ status = Column(String, default='new')
+
+ def __repr__(self):
+ if self.next_download or self.frequency:
+ return f''
+ return f''
+
+ def increment_next_download(self):
+ if self.frequency:
+ delta = timedelta(seconds=self.frequency)
+ else:
+ delta = DEFAULT_RETRY_FREQUENCY
+ self.next_download = now() + delta
+
+ def renew(self, reset_attempts: bool = False):
+ """
+ Mark this Download as "new" so it will be retried.
+ """
+ self.status = 'new'
+ if reset_attempts:
+ self.attempts = 0
+
+ def defer(self):
+ """
+ Download should be tried again after a time.
+ """
+ self.status = 'deferred'
+ self.increment_next_download()
+
+ def fail(self):
+ """
+ Download should not be attempted again. A recurring Download will raise an error.
+ """
+ if self.frequency:
+ raise ValueError('Recurring download should not be failed.')
+ self.status = 'failed'
+
+ def started(self):
+ """
+ Mark this Download as in progress.
+ """
+ self.attempts += 1
+ self.status = 'pending'
+
+ def complete(self):
+ """
+ Mark this Download as successfully downloaded.
+ """
+ self.status = 'complete'
+ self.last_successful_download = now()
+ if self.frequency:
+ self.increment_next_download()
+
+
+class Downloader:
+
+ def __init__(self, priority: int = 50):
+ """
+ Lower `priority` means the downloader will be checked first. Valid priority: 0-100
+
+ Downloaders of equal priority will be used at random.
+ """
+ if not 0 <= priority <= 100:
+ raise ValueError(f'Priority of {priority} of {self} is invalid!')
+
+ self.priority = priority
+ self._kill = multiprocessing.Event()
+
+ self._manager: DownloadManager = None # noqa
+
+ download_manager.register_downloader(self)
+
+ @staticmethod
+ def valid_url(url) -> bool:
+ raise NotImplementedError()
+
+ def do_download(self, download: Download):
+ raise NotImplementedError()
+
+ @property
+ def manager(self):
+ if self._manager is None:
+ raise NotImplementedError('This needs to be registered, see DownloadManager.register_downloader')
+ return self._manager
+
+ @manager.setter
+ def manager(self, value):
+ self._manager = value
+
+ def kill(self):
+ """
+ Kill the running download for this Downloader.
+ """
+ if not self._kill.is_set():
+ self._kill.set()
+
+ def clear(self):
+ """
+ Clear any "kill" request for this Downloader.
+ """
+ if self._kill.is_set():
+ self.clear()
+
+ def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> int:
+ """
+ Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
+ """
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, **kwargs)
+ pid = proc.pid
+ logger.debug(f'{self} launched download process {pid=} for {url}')
+
+ try:
+ while True:
+ try:
+ proc.wait(timeout=1.0)
+ # Download finished.
+ break
+ except subprocess.TimeoutExpired:
+ pass
+
+ if self._kill.is_set():
+ logger.warning(f'Killing download {pid=}')
+ proc.kill()
+ proc.poll()
+ raise UnrecoverableDownloadError(f'Download {pid=} was killed!')
+ finally:
+ self._kill.clear()
+
+ # Output all logs from the process.
+ # TODO is there a way to stream this output while the process is running?
+ logger.debug(f'Download exited with {proc.returncode}')
+ outs, errs = proc.communicate()
+ for line in outs.decode().splitlines(): # noqa
+ logger.debug(line)
+ for line in errs.decode().splitlines(): # noqa
+ logger.error(line)
+
+ return proc.returncode
+
+
+class DownloadManager:
+ priority_sorter = partial(sorted, key=attrgetter('priority'))
+
+ def __init__(self):
+ self.instances = tuple()
+
+ def register_downloader(self, instance: Downloader):
+ if not isinstance(instance, Downloader):
+ raise ValueError(f'Invalid downloader cannot be registered! {instance=}')
+ if instance in self.instances:
+ raise ValueError(f'Downloader already registered! {instance=}')
+
+ instance.manager = self
+
+ instances = (*self.instances, instance)
+ self.instances = tuple(self.priority_sorter(instances))
+
+ @cached(cache=TTLCache(maxsize=100, ttl=600))
+ def get_downloader(self, url: str):
+ for i in self.instances:
+ if i.valid_url(url):
+ return i
+ raise InvalidDownload(f'Invalid URL {url=}')
+
+ def valid_url(self, url: str) -> bool:
+ try:
+ self.get_downloader(url)
+ return True
+ except InvalidDownload:
+ return False
+
+ @staticmethod
+ def get_new_downloads(session: Session) -> List[Download]:
+ """
+ Get all "new" downloads. This method fetches the first download each iteration, so it will fetch downloads
+ that were created after calling it.
+ """
+ last = None
+ while True:
+ download = session.query(Download).filter_by(status='new').order_by(Download.id).first()
+ if not download:
+ return
+ if download == last:
+ # Got the last download again. Is something wrong?
+ return
+ last = download
+ yield download
+
+ def get_or_create_download(self, url: str, session: Session) -> Download:
+ """
+ Get a Download by its URL, if it cannot be found create one.
+ """
+ download = self.get_download(session, url=url)
+ if not download:
+ download = Download(url=url, status='new')
+ session.add(download)
+ session.flush()
+ return download
+
+ def create_download(self, url: str, session, skip_download: bool = False, reset_attempts: bool = False) -> Download:
+ """
+ Schedule a URL for download. If the URL failed previously, it will be retried.
+ """
+ # Make sure the URL can be downloaded.
+ self.get_downloader(url)
+
+ download = self.get_or_create_download(url, session)
+ # Download may have failed, try again.
+ download.renew(reset_attempts=reset_attempts)
+
+ session.commit()
+
+ if skip_download is True:
+ return download
+
+ if PYTEST:
+ # Download now (for testing).
+ self._do_downloads(session)
+ else:
+ # Start downloading ASAP.
+ self.start_downloads()
+
+ return download
+
+ def create_downloads(self, urls: List[str], session: Session = None, skip_download: bool = False,
+ reset_attempts: bool = False) -> List[Download]:
+ """
+ Schedule all URLs for download. If one cannot be downloaded, none will be added.
+ """
+ if not session:
+ _, session = get_db_context()
+
+ downloads = []
+
+ # Verify that all URLs can be downloaded.
+ for url in urls:
+ self.get_downloader(url)
+
+ with session.transaction:
+ for url in urls:
+ download = self.get_or_create_download(url, session)
+ # Download may have failed, try again.
+ download.renew(reset_attempts=reset_attempts)
+ downloads.append(download)
+
+ if skip_download is True:
+ return downloads
+
+ if PYTEST:
+ # Download now (for testing).
+ self._do_downloads(session)
+ else:
+ # Start downloading ASAP.
+ self.start_downloads()
+
+ return downloads
+
+ def _do_downloads(self, session=None):
+ # This is a long-running function, lets get a session that can be used for a long time.
+ if not session:
+ _, session = get_db_context()
+
+ try:
+ downloads = self.get_new_downloads(session)
+
+ if not downloads:
+ logger.info('No new downloads')
+ return
+
+ logger.info(f'Doing downloads.')
+ download_count = 0
+ for download in downloads:
+ download_count += 1
+ download_id = download.id
+
+ downloader = self.get_downloader(download.url)
+ downloader.clear()
+
+ download.started()
+ session.commit()
+
+ success = None
+ failure = None
+ try:
+ success = downloader.do_download(download)
+ except UnrecoverableDownloadError as e:
+ # Download failed and should not be retried.
+ logger.warning(f'UnrecoverableDownloadError for {download.url}', exc_info=e)
+ failure = True
+ except Exception as e:
+ logger.warning(f'Failed to download {download.url}. Will be tried again later.', exc_info=e)
+
+ # Long-running, get the download again.
+ with get_db_session(commit=True) as session_:
+ download = self.get_download(session_, id_=download_id)
+ if failure:
+ download.fail()
+ elif success:
+ download.complete()
+ else:
+ download.defer()
+ session_.commit()
+
+ logger.info(f'Done doing {download_count} downloads.')
+ finally:
+ session.close()
+
+ def do_downloads_sync(self):
+ acquired = DOWNLOAD_IN_PROGRESS.acquire(block=False)
+ if not acquired:
+ return
+
+ try:
+ self._do_downloads()
+ finally:
+ DOWNLOAD_IN_PROGRESS.release()
+
+ async def do_downloads(self):
+ return self.do_downloads_sync()
+
+ def start_downloads(self):
+ """
+ Start an async task to do downloads. This does nothing when testing!
+ """
+ if not PYTEST:
+ asyncio.create_task(self.do_downloads())
+
+ @staticmethod
+ def reset_downloads():
+ """
+ Set any incomplete Downloads to `new` so they will be retried.
+ """
+ with get_db_curs(commit=True) as curs:
+ curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
+
+ def recurring_download(self, url: str, frequency: int, skip_download: bool = False) -> Download:
+ """
+ Schedule a recurring download.
+ """
+ with get_db_session(commit=True) as session:
+ download = self.create_download(url, session=session, skip_download=True)
+ download.frequency = frequency
+
+ if skip_download is False and PYTEST:
+ self.do_downloads_sync()
+ elif skip_download is False:
+ self.start_downloads()
+
+ return download
+
+ DOWNLOAD_SORT = ('pending', 'failed', 'new', 'deferred', 'complete')
+
+ @classmethod
+ @iterify(list)
+ def _downloads_sorter(cls, downloads: List[Download]):
+ """
+ Downloads should be displayed to the user by status.
+ """
+ grouped_by_statuses = defaultdict(lambda: [])
+ for download in downloads:
+ grouped_by_statuses[download.status].append(download)
+
+ # Yield the downloads in the order defined above.
+ for sort in cls.DOWNLOAD_SORT:
+ yield from grouped_by_statuses[sort] # converted to a list by iterify()
+
+ def get_recurring_downloads(self, session: Session = None, limit: int = None):
+ """
+ Get all Downloads that will be downloaded in the future.
+ """
+ if not session:
+ _, session = get_db_context()
+ query = session.query(Download).filter(
+ Download.frequency != None # noqa
+ ).order_by(
+ Download.next_download, # Sort recurring by which will occur first, then by frequency.
+ Download.frequency, # Sort by frequency if the next_download is the same.
+ )
+ if limit:
+ query = query.limit(limit)
+ downloads = query.all()
+ downloads = self._downloads_sorter(downloads)
+ return downloads
+
+ def get_once_downloads(self, session: Session = None, limit: int = None):
+ """
+ Get all Downloads that will not reoccur.
+ """
+ if not session:
+ _, session = get_db_context()
+ query = session.query(Download).filter(
+ Download.frequency == None # noqa
+ ).order_by(
+ Download.last_successful_download.desc(),
+ Download.id,
+ )
+ if limit:
+ query = query.limit(limit)
+ downloads = query.all()
+ downloads = self._downloads_sorter(downloads)
+ return downloads
+
+ def renew_recurring_downloads(self, session: Session = None):
+ """
+ Mark any recurring downloads that are due for download as "new". Start a download.
+ """
+ now_ = now()
+ if not session:
+ _, session = get_db_context()
+
+ recurring = self.get_recurring_downloads(session)
+ renewed = False
+ for download in recurring:
+ if download.next_download <= now_:
+ download.renew()
+ renewed = True
+
+ if renewed:
+ session.commit()
+ self.start_downloads()
+
+ @staticmethod
+ def get_downloads(session: Session) -> List[Download]:
+ downloads = session.query(Download).all()
+ return list(downloads)
+
+ @staticmethod
+ def get_download(session: Session, url: str = None, id_: int = None) -> Optional[Download]:
+ """
+ Attempt to find a Download by its URL or by its id.
+ """
+ query = session.query(Download)
+ if url:
+ return query.filter_by(url=url).one_or_none()
+ elif id:
+ return query.filter_by(id=id_).one_or_none()
+
+ def delete_download(self, download_id: int, session: Session = None):
+ """
+ Delete a Download. Returns True if a Download was deleted, otherwise return False.
+ """
+ if not session:
+ _, session = get_db_context()
+ download = self.get_download(session, id_=download_id)
+ if download:
+ session.delete(download)
+ session.commit()
+ return True
+ return False
+
+ def kill_download(self, download_id: int):
+ """
+ Fail a Download, if it is pending, kill the Downloader so the download stops.
+ """
+ with get_db_session(commit=True) as session:
+ download = self.get_download(session, id_=download_id)
+ downloader = self.get_downloader(download.url)
+ logger.warning(f'Killing download {download_id} in {downloader}')
+ if download.status == 'pending':
+ downloader.kill()
+ download.fail()
+
+ FINISHED_STATUSES = ('complete', 'failed')
+
+ def delete_old_once_downloads(self):
+ """
+ Delete all once-downloads that have expired. Do not delete downloads that are new, or should be tried again.
+ """
+ with get_db_session(commit=True) as session:
+ downloads = self.get_once_downloads(session)
+ one_month = now() - timedelta(days=30)
+ for download in downloads:
+ if download.status in self.FINISHED_STATUSES and download.last_successful_download < one_month:
+ session.delete(download)
+
+
+# The global DownloadManager. This should be used everywhere!
+download_manager = DownloadManager()
diff --git a/wrolpi/errors.py b/wrolpi/errors.py
index f27875a8..eda916f0 100644
--- a/wrolpi/errors.py
+++ b/wrolpi/errors.py
@@ -36,6 +36,10 @@ class ChannelDirectoryConflict(APIError):
pass
+class ChannelSourceIdConflict(APIError):
+ pass
+
+
class SearchEmpty(APIError):
pass
@@ -124,6 +128,14 @@ class InvalidArchive(APIError):
pass
+class InvalidDownload(APIError):
+ pass
+
+
+class UnrecoverableDownloadError(APIError):
+ pass
+
+
error_codes = iter(range(1, 1000))
API_ERRORS = {
@@ -272,4 +284,19 @@ API_ERRORS = {
'message': 'The archive is invalid. See server logs.',
'status': HTTPStatus.INTERNAL_SERVER_ERROR,
},
+ InvalidDownload: {
+ 'code': next(error_codes),
+ 'message': 'The URL cannot be downloaded.',
+ 'status': HTTPStatus.BAD_REQUEST,
+ },
+ ChannelSourceIdConflict: {
+ 'code': next(error_codes),
+ 'message': 'Channel with source id already exists',
+ 'status': HTTPStatus.INTERNAL_SERVER_ERROR,
+ },
+ UnrecoverableDownloadError: {
+ 'code': next(error_codes),
+ 'message': 'Download experienced an error which cannot be recovered. Download will be deleted.',
+ 'status': HTTPStatus.INTERNAL_SERVER_ERROR,
+ }
}
diff --git a/wrolpi/media_path.py b/wrolpi/media_path.py
index 31209c90..71a4a198 100644
--- a/wrolpi/media_path.py
+++ b/wrolpi/media_path.py
@@ -51,6 +51,12 @@ class MediaPath:
def __eq__(self, other):
return self._path == other
+ def __truediv__(self, other):
+ return self._path / other
+
+ def startswith(self, other):
+ return str(self._path).startswith(other)
+
class MediaPathType(types.TypeDecorator): # noqa
impl = types.String
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 7c80a2ba..38305ad2 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -1,5 +1,6 @@
import json
-import logging
+import json
+import multiprocessing
import re
from datetime import datetime, date
from decimal import Decimal
@@ -8,18 +9,20 @@ from http import HTTPStatus
from pathlib import Path
from pytz import UnknownTimeZoneError
-from sanic import Sanic, response, Blueprint
+from sanic import Sanic, response, Blueprint, __version__ as sanic_version
from sanic.request import Request
from sanic.response import HTTPResponse
from sanic_cors import CORS
-from sanic_openapi import swagger_blueprint
-from wrolpi.common import EVENTS, set_sanic_url_parts, logger, get_config, wrol_mode_enabled, save_settings_config, \
- Base
+from wrolpi.common import set_sanic_url_parts, logger, get_config, wrol_mode_enabled, save_settings_config, \
+ Base, get_media_directory, wrol_mode_check
from wrolpi.dates import set_timezone
+from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, InvalidTimezone
+from wrolpi.media_path import MediaPath
from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, EchoResponse, \
- EventsResponse, validate_doc
+ validate_doc, DownloadRequest
+from wrolpi.vars import DOCKERIZED
logger = logger.getChild(__name__)
@@ -38,10 +41,9 @@ CORS(
}
)
-root_api = Blueprint('Root API', url_prefix='/api')
+root_api = Blueprint('RootAPI', url_prefix='/api')
-# Attach the Sanic OpenAPI blueprint to the API App. This will generate our API docs.
-BLUEPRINTS = [swagger_blueprint, root_api, ]
+BLUEPRINTS = [root_api, ]
def get_blueprint(name: str, url_prefix: str) -> Blueprint:
@@ -57,29 +59,29 @@ def add_blueprint(bp: Blueprint):
BLUEPRINTS.append(bp)
-def run_webserver(host: str, port: int, workers: int = 8):
+def run_webserver(loop, host: str, port: int, workers: int = 8):
set_sanic_url_parts(host, port)
# Attach all blueprints after they have been defined.
for bp in BLUEPRINTS:
api_app.blueprint(bp)
- # Sanic should match our logging level.
- debug = logger.level == logging.DEBUG
# TODO remove the auto reload when development is stable
- api_app.run(host, port, workers=workers, debug=debug, auto_reload=True)
+ kwargs = dict(host=host, port=port, workers=workers, auto_reload=DOCKERIZED)
+ logger.debug(f'Running Sanic {sanic_version} with kwargs {kwargs}')
+ return api_app.run(**kwargs)
def init_parser(parser):
# Called by WROLPI's main() function
parser.add_argument('-H', '--host', default=DEFAULT_HOST, help='What network interface to connect webserver')
parser.add_argument('-p', '--port', default=DEFAULT_PORT, type=int, help='What port to connect webserver')
- parser.add_argument('-w', '--workers', default=4, type=int, help='How many web workers to run')
+ parser.add_argument('-w', '--workers', default=multiprocessing.cpu_count(), type=int,
+ help='How many web workers to run')
-def main(args):
- run_webserver(args.host, args.port, args.workers)
- return 0
+def main(loop, args):
+ return run_webserver(loop, args.host, args.port, args.workers)
index_html = '''
@@ -122,16 +124,6 @@ async def echo(request: Request):
return response.json(ret)
-@root_api.get('/events')
-@validate_doc(
- summary='Get a list of event feeds',
- produces=EventsResponse,
-)
-def events(_: Request):
- e = [{'name': name, 'is_set': event.is_set()} for (name, event) in EVENTS]
- return response.json({'events': e})
-
-
@root_api.route('/settings', methods=['GET', 'OPTIONS'])
@validate_doc(
summary='Get WROLPi settings',
@@ -180,23 +172,72 @@ def valid_regex(_: Request, data: dict):
return response.json({'valid': False, 'regex': data['regex']}, HTTPStatus.BAD_REQUEST)
+@root_api.post('/download')
+@validate_doc(
+ summary='Download the many URLs that are provided.',
+ consumes=DownloadRequest,
+)
+@wrol_mode_check
+async def post_download(request: Request, data: dict):
+ # URLs are provided in a textarea, lets split all lines.
+ urls = [i.strip() for i in str(data['urls']).strip().splitlines()]
+ download_manager.create_downloads(urls)
+ return response.empty()
+
+
+@root_api.get('/download')
+@validate_doc(
+ summary='Get all Downloads that need to be processed.',
+)
+async def get_downloads(request: Request):
+ data = dict(
+ recurring_downloads=download_manager.get_recurring_downloads(),
+ once_downloads=download_manager.get_once_downloads(limit=1000),
+ )
+ return json_response(data)
+
+
+@root_api.post('/download//kill')
+async def kill_download(request: Request, download_id: int):
+ download_manager.kill_download(download_id)
+ return response.empty()
+
+
+@root_api.delete('/download/')
+@wrol_mode_check
+async def delete_download(request: Request, download_id: int):
+ deleted = download_manager.delete_download(download_id)
+ return response.empty(HTTPStatus.NO_CONTENT if deleted else HTTPStatus.NOT_FOUND)
+
+
class CustomJSONEncoder(json.JSONEncoder):
def default(self, obj):
- if isinstance(obj, datetime):
- return obj.timestamp()
- elif isinstance(obj, date):
- return datetime(obj.year, obj.month, obj.day).timestamp()
- elif isinstance(obj, Decimal):
- return str(obj)
- elif isinstance(obj, Base):
- if hasattr(obj, 'dict'):
- return obj.dict()
- elif isinstance(obj, Path):
- return str(obj)
- elif hasattr(obj, '__json__'):
- return obj.__json__()
- return super(CustomJSONEncoder, self).default(obj)
+ try:
+ if hasattr(obj, '__json__'):
+ # Get __json__ before others.
+ return obj.__json__()
+ elif isinstance(obj, datetime):
+ return obj.timestamp()
+ elif isinstance(obj, date):
+ return datetime(obj.year, obj.month, obj.day).timestamp()
+ elif isinstance(obj, Decimal):
+ return str(obj)
+ elif isinstance(obj, Base):
+ if hasattr(obj, 'dict'):
+ return obj.dict()
+ elif isinstance(obj, Path):
+ return str(obj)
+ elif isinstance(obj, MediaPath):
+ media_directory = get_media_directory()
+ path = obj.path.relative_to(media_directory)
+ if str(path) == '.':
+ return ''
+ return str(path)
+ return super(CustomJSONEncoder, self).default(obj)
+ except Exception as e:
+ logger.fatal(f'Failed to JSON encode {obj}', exc_info=e)
+ raise
@wraps(response.json)
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index 754f8966..f0e37514 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -178,6 +178,10 @@ class EventsResponse:
events = doc.List(EventObject)
+class DownloadRequest:
+ urls = doc.String()
+
+
class Trinary(Field):
"""
A field for API docs. Can be True/False/None.
diff --git a/wrolpi/test/common.py b/wrolpi/test/common.py
index 3a54bf00..37aec451 100644
--- a/wrolpi/test/common.py
+++ b/wrolpi/test/common.py
@@ -6,54 +6,22 @@ from contextlib import contextmanager
from functools import partialmethod
from http import HTTPStatus
from itertools import zip_longest
-from queue import Empty
from shutil import copyfile
-from typing import List, Tuple, Optional
-from uuid import uuid1
+from typing import List, Optional
import mock
import websockets
import yaml
from sanic_openapi.api import Response
-from sqlalchemy import create_engine
-from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, sessionmaker
-from wrolpi.common import EXAMPLE_CONFIG_PATH, get_config, Base, QUEUES, set_test_media_directory
-from wrolpi.db import postgres_engine, get_db_args
-from wrolpi.root_api import BLUEPRINTS, api_app
+from wrolpi.common import EXAMPLE_CONFIG_PATH, get_config, set_test_media_directory, get_media_directory
+from wrolpi.conftest import ROUTES_ATTACHED, test_db, test_client # noqa
+from wrolpi.db import postgres_engine
from wrolpi.vars import PROJECT_DIR
TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False)
-def get_test_db_engine():
- suffix = str(uuid1()).replace('-', '')
- db_name = f'wrolpi_testing_{suffix}'
- conn = postgres_engine.connect()
- conn.execute(f'DROP DATABASE IF EXISTS {db_name}')
- conn.execute(f'CREATE DATABASE {db_name}')
- conn.execute('commit')
-
- test_args = get_db_args(db_name)
- test_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**test_args))
- return test_engine
-
-
-def test_db() -> Tuple[Engine, Session]:
- """
- Create a unique SQLAlchemy engine/session for a test.
- """
- test_engine = get_test_db_engine()
- if test_engine.engine.url.database == 'wrolpi':
- raise ValueError('Refusing the test on wrolpi database!')
-
- # Create all tables. No need to check if they exist because this is a test DB.
- Base.metadata.create_all(test_engine, checkfirst=False)
- session = sessionmaker(bind=test_engine)()
- return test_engine, session
-
-
def wrap_test_db(func):
"""
Wrap a test so that when calling wrolpi.common.get_db, it returns a testing database cloned from the api
@@ -63,12 +31,12 @@ def wrap_test_db(func):
def wrapped(*a, **kw):
test_engine, session = test_db()
- def fake_get_db_context():
+ def fake_get_db_session():
"""Get the testing db"""
return test_engine, session
try:
- with mock.patch('wrolpi.db._get_db_session', fake_get_db_context):
+ with mock.patch('wrolpi.db._get_db_session', fake_get_db_session):
# Run the test.
result = func(*a, **kw)
return result
@@ -82,44 +50,32 @@ def wrap_test_db(func):
return wrapped
-def get_all_messages_in_queue(q):
- """Get all messages in a Queue without waiting."""
- messages = []
- while True:
- try:
- msg = q.get_nowait()
- messages.append(msg)
- except Empty:
- break
- return messages
-
-
class PytestCase:
"""
Replicate unittest.TestCase methods to bridge the gap between it and Pytest.
"""
@staticmethod
- def assertGreater(a, b, msg: str = None):
+ def assertGreater(a, b, msg: str = None): # noqa
assert a > b, msg
@staticmethod
- def assertLess(a, b, msg: str = None):
+ def assertLess(a, b, msg: str = None): # noqa
assert a < b, msg
@staticmethod
- def assertEqual(a, b, msg: str = None):
+ def assertEqual(a, b, msg: str = None): # noqa
assert a == b, msg or f'{a} != {b}'
@staticmethod
- def assertRaises(exception, func, *args, **kwargs):
+ def assertRaises(exception, func, *args, **kwargs): # noqa
try:
func(*args, **kwargs)
except exception:
pass
@classmethod
- def assertDictContains(cls, d1: dict, d2: dict):
+ def assertDictContains(cls, d1: dict, d2: dict): # noqa
if hasattr(d1, '__dict__'):
d1 = d1.__dict__
if hasattr(d2, '__dict__'):
@@ -134,13 +90,13 @@ class PytestCase:
else:
assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'
- def assertError(self, response, http_status: int, code=None):
+ def assertError(self, response, http_status: int, code=None): # noqa
self.assertEqual(response.status_code, http_status)
if code:
self.assertEqual(response.json['code'], code)
@staticmethod
- def assertTruth(value, expected):
+ def assertTruth(value, expected): # noqa
"""
Check that a value is Truthy or Falsy.
"""
@@ -161,6 +117,10 @@ class PytestCase:
raise ValueError('d2 is None')
self.assertTruth(d1[d2_key], d2[d2_key])
+ @staticmethod
+ def assertLength(a, b): # noqa
+ assert len(a) == len(b), f'{len(a)=} != {len(b)=}'
+
class ExtendedTestCase(PytestCase, unittest.TestCase):
"""
@@ -169,9 +129,6 @@ class ExtendedTestCase(PytestCase, unittest.TestCase):
pass
-ROUTES_ATTACHED = False
-
-
class TestAPI(ExtendedTestCase):
def setUp(self) -> None:
@@ -188,21 +145,9 @@ class TestAPI(ExtendedTestCase):
with open(TEST_CONFIG_PATH.name, 'wt') as fh:
fh.write(yaml.dump(config))
- @classmethod
- def setUpClass(cls) -> None:
- global ROUTES_ATTACHED
- if ROUTES_ATTACHED is False:
- # Attach any blueprints for the test.
- for bp in BLUEPRINTS:
- api_app.blueprint(bp)
- ROUTES_ATTACHED = True
-
def tearDown(self) -> None:
self.config_path_patch.stop()
set_test_media_directory(None)
- # Clear out any messages in queues
- for q in QUEUES:
- get_all_messages_in_queue(q)
def assertHTTPStatus(self, response: Response, status: int):
self.assertEqual(response.status_code, status)
@@ -229,7 +174,7 @@ def wrap_media_directory(path: Optional[pathlib.Path] = None):
@contextmanager
-def build_test_directories(paths: List[str]) -> pathlib.Path:
+def build_test_directories(paths: List[str], tmp_dir: pathlib.Path = None) -> pathlib.Path:
"""
Create directories based on the provided structure.
@@ -247,23 +192,20 @@ def build_test_directories(paths: List[str]) -> pathlib.Path:
channel2/vid2.mp4
channel2/vid2.en.vtt
"""
- dir_ = get_config().get('media_directory')
- dir_ = pathlib.Path(dir_).absolute()
- with tempfile.TemporaryDirectory(dir=dir_) as temp_dir:
- root = pathlib.Path(temp_dir)
+ tmp_dir = tmp_dir or get_media_directory()
- directories = filter(lambda i: i.endswith('/'), paths)
- for directory in directories:
- (root / directory).mkdir(parents=True)
+ directories = filter(lambda i: i.endswith('/'), paths)
+ for directory in directories:
+ (tmp_dir / directory).mkdir(parents=True)
- files = filter(lambda i: not i.endswith('/'), paths)
- for file in files:
- file = root / file
- parents = file.parents
- parents[0].mkdir(parents=True, exist_ok=True)
- (root / file).touch()
+ files = filter(lambda i: not i.endswith('/'), paths)
+ for file in files:
+ file = tmp_dir / file
+ parents = file.parents
+ parents[0].mkdir(parents=True, exist_ok=True)
+ (tmp_dir / file).touch()
- yield root.absolute()
+ yield tmp_dir.absolute()
async def get_all_ws_messages(ws) -> List[dict]:
diff --git a/wrolpi/test/test_api.py b/wrolpi/test/test_api.py
deleted file mode 100644
index e6c71394..00000000
--- a/wrolpi/test/test_api.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import json
-from http import HTTPStatus
-from unittest import mock
-from unittest.mock import MagicMock
-
-import pytest
-from sanic_openapi import swagger_blueprint
-
-from wrolpi.root_api import api_app
-from wrolpi.test.common import TestAPI
-
-
-class TestRootAPI(TestAPI):
-
- def test_index(self):
- """
- Index should have some details in an HTML response
- """
- request, response = api_app.test_client.get('/')
- assert response.status_code == HTTPStatus.OK
- assert b'html' in response.body
-
- def test_echo(self):
- """
- Echo should send back a JSON object with our request details.
- """
- request, response = api_app.test_client.get('/api/echo?hello=world')
- assert response.status_code == HTTPStatus.OK
- self.assertDictContains(response.json['form'], {})
- assert response.json['method'] == 'GET', 'Method was not GET'
- assert response.json['json'] is None, 'JSON was not empty'
- assert response.json['args'] == {'hello': ['world']}
-
- data = {'foo': 'bar'}
- request, response = api_app.test_client.post('/api/echo', data=json.dumps(data))
- assert response.status_code == HTTPStatus.OK
- self.assertDictContains(response.json['form'], {})
- assert response.json['method'] == 'POST', 'Method was not POST'
- assert response.json['json'] == data, 'JSON was not data'
- assert response.json['args'] == {}
-
- def test_valid_regex(self):
- """
- The endpoint should only return valid if the regex is valid.
- """
- data = {'regex': 'foo'}
- request, response = api_app.test_client.post('/api/valid_regex', data=json.dumps(data))
- assert response.status_code == HTTPStatus.OK
- assert json.loads(response.body) == {'valid': True, 'regex': 'foo'}
-
- data = {'regex': '.*(title match).*'}
- request, response = api_app.test_client.post('/api/valid_regex', data=json.dumps(data))
- assert response.status_code == HTTPStatus.OK
- assert json.loads(response.body) == {'valid': True, 'regex': '.*(title match).*'}
-
- data = {'regex': '.*(missing parenthesis.*'}
- request, response = api_app.test_client.post('/api/valid_regex', data=json.dumps(data))
- assert response.status_code == HTTPStatus.BAD_REQUEST
- assert json.loads(response.body) == {'valid': False, 'regex': '.*(missing parenthesis.*'}
-
- @pytest.mark.xfail
- def test_swagger(self):
- """
- Swagger can be generated. This test is just to assure that our spec generation hasn't broken.
- """
- # An API request is required to set things up (apparently).
- api_app.test_client.get('/')
-
- from sanic.response import json as json_
- json_(swagger_blueprint._spec)
-
- def test_get_settings(self):
- request, response = api_app.test_client.get('/api/settings')
- self.assertOK(response)
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index 9cea93a0..579bf08e 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -1,15 +1,14 @@
import unittest
from datetime import date, datetime
-from queue import Queue
import pytest
from sanic_openapi import doc
-from wrolpi.common import combine_dicts, insert_parameter, ProgressReporter, date_range
+from wrolpi.common import combine_dicts, insert_parameter, date_range
from wrolpi.dates import set_timezone, now
from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone
from wrolpi.schema import validate_data
-from wrolpi.test.common import build_test_directories, wrap_test_db
+from wrolpi.test.common import build_test_directories, wrap_media_directory
class Model:
@@ -114,42 +113,43 @@ def test_combine_dicts(data, expected):
def test_build_video_directories():
- structure = [
- 'channel1/vid1.mp4',
- ]
- with build_test_directories(structure) as tempdir:
- assert (tempdir / 'channel1').is_dir()
- assert (tempdir / 'channel1/vid1.mp4').is_file()
+ with wrap_media_directory():
+ structure = [
+ 'channel1/vid1.mp4',
+ ]
+ with build_test_directories(structure) as tempdir:
+ assert (tempdir / 'channel1').is_dir()
+ assert (tempdir / 'channel1/vid1.mp4').is_file()
- structure = [
- 'channel2/',
- 'channel2.1/channel2.2/',
- ]
- with build_test_directories(structure) as tempdir:
- assert (tempdir / 'channel2').is_dir()
- assert (tempdir / 'channel2.1/channel2.2').is_dir()
+ structure = [
+ 'channel2/',
+ 'channel2.1/channel2.2/',
+ ]
+ with build_test_directories(structure) as tempdir:
+ assert (tempdir / 'channel2').is_dir()
+ assert (tempdir / 'channel2.1/channel2.2').is_dir()
- structure = [
- 'channel3/vid1.mp4',
- 'channel3/vid2.mp4',
- 'channel4/vid1.mp4',
- 'channel4/vid1.en.vtt',
- 'channel5/',
- ]
- with build_test_directories(structure) as tempdir:
- assert (tempdir / 'channel3/vid1.mp4').is_file()
- assert (tempdir / 'channel3').is_dir()
- assert (tempdir / 'channel3/vid2.mp4').is_file()
- assert (tempdir / 'channel4/vid1.mp4').is_file()
- assert (tempdir / 'channel4/vid1.en.vtt').is_file()
- assert (tempdir / 'channel5').is_dir()
+ structure = [
+ 'channel3/vid1.mp4',
+ 'channel3/vid2.mp4',
+ 'channel4/vid1.mp4',
+ 'channel4/vid1.en.vtt',
+ 'channel5/',
+ ]
+ with build_test_directories(structure) as tempdir:
+ assert (tempdir / 'channel3/vid1.mp4').is_file()
+ assert (tempdir / 'channel3').is_dir()
+ assert (tempdir / 'channel3/vid2.mp4').is_file()
+ assert (tempdir / 'channel4/vid1.mp4').is_file()
+ assert (tempdir / 'channel4/vid1.en.vtt').is_file()
+ assert (tempdir / 'channel5').is_dir()
- structure = [
- 'channel6/subdirectory/vid1.mp4',
- ]
- with build_test_directories(structure) as tempdir:
- assert (tempdir / 'channel6/subdirectory').is_dir()
- assert (tempdir / 'channel6/subdirectory/vid1.mp4').is_file()
+ structure = [
+ 'channel6/subdirectory/vid1.mp4',
+ ]
+ with build_test_directories(structure) as tempdir:
+ assert (tempdir / 'channel6/subdirectory').is_dir()
+ assert (tempdir / 'channel6/subdirectory/vid1.mp4').is_file()
def test_insert_parameter():
@@ -195,146 +195,6 @@ def test_insert_parameter():
pytest.raises(TypeError, insert_parameter, func, 'bar', 'bar', (1,), {})
-@pytest.mark.parametrize(
- 'totals,progresses,messages',
- [
- (
- [25],
- [],
- [
- {'progresses': [dict(message='foo', percent=0, total=25, value=0)]},
- {'code': 'bar', 'progresses': [dict(message='foo', percent=0, total=25, value=0)]},
- {'code': 'error', 'progresses': [dict(message='baz', percent=0, total=25, value=0)]},
- ],
- ),
- (
- [25],
- [
- (0, 1),
- (0, 2),
- (0, 4),
- (0, 8),
- (0, 16),
- (0, 25),
- ],
- [
- {'progresses': [dict(message='foo', percent=0, total=25, value=0)]},
- {'code': 'bar', 'progresses': [dict(message='foo', percent=0, total=25, value=0)]},
- {'progresses': [dict(message='foo', percent=4, total=25, value=1)]},
- {'progresses': [dict(message='foo', percent=8, total=25, value=2)]},
- {'progresses': [dict(message='foo', percent=16, total=25, value=4)]},
- {'progresses': [dict(message='foo', percent=32, total=25, value=8)]},
- {'progresses': [dict(message='foo', percent=64, total=25, value=16)]},
- {'progresses': [dict(message='foo', percent=100, total=25, value=25)]},
- {'code': 'error', 'progresses': [dict(message='baz', percent=100, total=25, value=25)]},
- ]
- ),
- (
- [25, 10],
- [
- (1, 3),
- (1, 8),
- (0, 18),
- (1, 10),
- (0, 30), # This is higher than the total of 25, percent should be 100.
- ],
- [
- {'progresses': [
- dict(message='foo', percent=0, total=25, value=0),
- dict(percent=0, total=10, value=0),
- ]},
- {'code': 'bar', 'progresses': [
- dict(message='foo', percent=0, total=25, value=0),
- dict(percent=0, total=10, value=0),
- ]},
- {'progresses': [
- dict(message='foo', percent=0, total=25, value=0),
- dict(percent=30, total=10, value=3),
- ]},
- {'progresses': [
- dict(message='foo', percent=0, total=25, value=0),
- dict(percent=80, total=10, value=8),
- ]},
- {'progresses': [
- dict(message='foo', percent=72, total=25, value=18),
- dict(percent=80, total=10, value=8),
- ]},
- {'progresses': [
- dict(message='foo', percent=72, total=25, value=18),
- dict(percent=100, total=10, value=10),
- ]},
- {'progresses': [
- dict(message='foo', percent=100, total=25, value=30),
- dict(percent=100, total=10, value=10),
- ]},
- {
- 'code': 'error',
- 'progresses': [
- dict(message='baz', percent=100, total=25, value=30),
- dict(percent=100, total=10, value=10),
- ]
- },
- ]
- ),
- ]
-)
-def test_feed_reporter(totals, progresses, messages):
- q = Queue()
- reporter = ProgressReporter(q, len(totals))
-
- for idx, total in enumerate(totals):
- reporter.set_progress_total(idx, total)
-
- reporter.message(0, 'foo')
- reporter.code('bar')
-
- for progress in progresses:
- reporter.send_progress(*progress)
-
- reporter.error(0, 'baz')
-
- count = 0
- while not q.empty():
- received = q.get_nowait()
- try:
- message = messages.pop(0)
- except IndexError:
- raise AssertionError(f'Queue ({count}) had excess message: {received}')
-
- assert message == received, f'Message {count} did not match'
-
- count += 1
-
-
-def test_feed_reporter_finish():
- q = Queue()
- reporter = ProgressReporter(q)
-
- reporter.set_progress_total(0, 50)
- reporter.send_progress(0, 0)
- reporter.send_progress(0, 20)
- reporter.finish(0, 'completed')
-
- expected = [
- {'progresses': [{'percent': 0, 'total': 50, 'value': 0}]},
- {'progresses': [{'percent': 40, 'total': 50, 'value': 20}]},
- {'progresses': [{'message': 'completed', 'percent': 100, 'value': 50, 'total': 50}]},
- ]
-
- count = 0
-
- while not q.empty():
- received = q.get_nowait()
- try:
- message = expected.pop(0)
- except IndexError:
- raise AssertionError(f'Queue ({count}) had excess message: {received}')
-
- assert message == received, f'Message {count} did not match'
-
- count += 1
-
-
class TestCommon(unittest.TestCase):
def test_date_range(self):
diff --git a/wrolpi/test/test_dates.py b/wrolpi/test/test_dates.py
index cb011016..30bbd921 100644
--- a/wrolpi/test/test_dates.py
+++ b/wrolpi/test/test_dates.py
@@ -1,8 +1,10 @@
+from datetime import datetime, timedelta
+
from sqlalchemy import Column, Integer
from wrolpi.common import Base
-from wrolpi.dates import TZDateTime
-from wrolpi.db import get_db_session
+from wrolpi.dates import TZDateTime, local_timezone, strftime_ms
+from wrolpi.db import get_db_session, get_db_curs
from wrolpi.test.common import TestAPI, wrap_test_db
@@ -17,11 +19,19 @@ class TestTable(Base):
class TestDates(TestAPI):
+ def assert_raw_datetime(self, expected_datetime: str):
+ with get_db_curs() as curs:
+ curs.execute('SELECT * FROM test_table')
+ dt_ = curs.fetchone()['dt']
+ self.assertEqual(strftime_ms(dt_), expected_datetime)
+
@wrap_test_db
def test_TZDateTime(self):
with get_db_session(commit=True) as session:
# TZDateTime can be None
- session.add(TestTable())
+ tt = TestTable()
+ session.add(tt)
+ self.assertIsNone(tt.dt)
with get_db_session(commit=True) as session:
tt = session.query(TestTable).one()
@@ -30,14 +40,22 @@ class TestDates(TestAPI):
# Store a timezone naive string, it should be assumed this is a local time.
tt.dt = '2021-10-05 16:20:10.346823'
+ # DB actually contains a UTC timestamp.
+ self.assert_raw_datetime('2021-10-05 22:20:10.346823')
+
with get_db_session(commit=True) as session:
tt = session.query(TestTable).one()
- self.assertEqual(str(tt.dt), '2021-10-05 16:20:10.346823-06:00')
+ # Datetime is unchanged and localized.
+ self.assertEqual(tt.dt, local_timezone(datetime(2021, 10, 5, 16, 20, 10, 346823)))
- # Store a timezone string.
- tt.dt = '2021-10-05 00:20:10.346824-08:00'
+ # Increment by one hour.
+ tt.dt += timedelta(seconds=60 * 60)
+ self.assertEqual(tt.dt, local_timezone(datetime(2021, 10, 5, 17, 20, 10, 346823)))
- with get_db_session() as session:
+ # DB is incremented by one hour.
+ self.assert_raw_datetime('2021-10-05 23:20:10.346823')
+
+ with get_db_session(commit=True) as session:
tt = session.query(TestTable).one()
- # Timezone is preserved.
- self.assertEqual(str(tt.dt), '2021-10-05 08:20:10.346824-06:00')
+ # Datetime is incremented by an hour.
+ self.assertEqual(tt.dt, local_timezone(datetime(2021, 10, 5, 17, 20, 10, 346823)))
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
new file mode 100644
index 00000000..8f5b53df
--- /dev/null
+++ b/wrolpi/test/test_downloader.py
@@ -0,0 +1,303 @@
+from datetime import datetime, timedelta
+from itertools import zip_longest
+from unittest import mock
+from unittest.mock import MagicMock
+
+from wrolpi import downloader
+from wrolpi.dates import local_timezone
+from wrolpi.db import get_db_context
+from wrolpi.downloader import Downloader, Download
+from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload
+from wrolpi.test.common import wrap_test_db, TestAPI
+
+
+class PermissiveDownloader(Downloader):
+ """
+ A testing Downloader which always says it's valid.
+ """
+
+ def valid_url(self, url: str):
+ return True
+
+
+class HTTPDownloader(Downloader):
+ """
+ A testing Downloader which says its valid when a URL starts with http/https
+ """
+
+ def valid_url(self, url: str):
+ return url.startswith('https://') or url.startswith('http://')
+
+
+class TestDownloader(TestAPI):
+ def setUp(self) -> None:
+ super().setUp()
+ self.mgr = downloader.DownloadManager()
+ PermissiveDownloader.do_download = MagicMock()
+ HTTPDownloader.do_download = MagicMock()
+
+ def test_register_downloader(self):
+ """
+ Downloaders can be registered and have specific priorities.
+ """
+ self.assertEqual(self.mgr.instances, tuple())
+
+ http_downloader = HTTPDownloader()
+ self.mgr.register_downloader(http_downloader)
+ self.assertEqual(self.mgr.instances, (http_downloader,))
+
+ # PermissiveDownloader is first priority.
+ permissive_downloader = PermissiveDownloader(priority=0)
+ self.mgr.register_downloader(permissive_downloader)
+ self.assertEqual(self.mgr.instances, (permissive_downloader, http_downloader))
+
+ self.assertRaises(ValueError, self.mgr.register_downloader, http_downloader)
+ self.assertRaises(ValueError, self.mgr.register_downloader, permissive_downloader)
+
+ def test_valid_url(self):
+ # No downloaders available.
+ self.assertFalse(self.mgr.valid_url('foo'))
+ self.assertFalse(self.mgr.valid_url('https://example.com'))
+
+ http_downloader = HTTPDownloader()
+ self.mgr.register_downloader(http_downloader)
+ self.assertFalse(self.mgr.valid_url('foo'))
+ self.assertTrue(self.mgr.valid_url('https://example.com'))
+ self.assertEqual(self.mgr.get_downloader('https://example.com'), http_downloader)
+
+ # Last priority.
+ permissive_downloader = PermissiveDownloader(priority=100)
+ self.mgr.register_downloader(permissive_downloader)
+ self.assertTrue(self.mgr.valid_url('foo'))
+ self.assertEqual(self.mgr.get_downloader('foo'), permissive_downloader)
+ self.assertTrue(self.mgr.valid_url('https://example.com'))
+ self.assertEqual(self.mgr.get_downloader('https://example.com'), http_downloader)
+
+ @wrap_test_db
+ def test_ensure_download(self):
+ _, session = get_db_context()
+
+ self.assertEqual(len(self.mgr.get_downloads(session)), 0)
+
+ http_downloader = HTTPDownloader()
+ http_downloader.do_download.return_value = False
+ self.mgr.register_downloader(http_downloader)
+
+ self.mgr.create_download('https://example.com', session)
+ download = self.mgr.get_download(session, 'https://example.com')
+ self.assertEqual(download.url, 'https://example.com')
+
+ self.assertEqual(len(self.mgr.get_downloads(session)), 1)
+ self.assertIsNotNone(self.mgr.get_download(session, 'https://example.com'))
+ self.assertIsNone(self.mgr.get_download(session, 'https://example.com/not downloading'))
+
+ @wrap_test_db
+ def test_do_downloads(self):
+ _, session = get_db_context()
+
+ http_downloader = HTTPDownloader()
+ http_downloader.do_download.return_value = True
+ self.mgr.register_downloader(http_downloader)
+
+ permissive_downloader = PermissiveDownloader(priority=100)
+ permissive_downloader.do_download.return_value = False # returns a failure
+ self.mgr.register_downloader(permissive_downloader)
+
+ # https is handled by the HTTP Downloader.
+ self.mgr.create_download('https://example.com', session)
+ http_downloader.do_download.assert_called_once()
+ permissive_downloader.do_download.assert_not_called()
+ self.assertIsNotNone(self.mgr.get_download(session, 'https://example.com'))
+
+ http_downloader.do_download.reset_mock()
+
+ # try the permissive download, which returns a failure.
+ self.mgr.create_download('foo', session)
+ http_downloader.do_download.assert_not_called()
+ permissive_downloader.do_download.assert_called_once()
+ download = self.mgr.get_download(session, 'foo')
+ self.assertEqual(download.attempts, 1)
+
+ # try again
+ self.mgr.create_download('foo', session)
+ self.mgr._do_downloads(session)
+ download = self.mgr.get_download(session, 'foo')
+ self.assertEqual(download.attempts, 2)
+
+ # finally success
+ permissive_downloader.do_download.return_value = True
+ self.mgr.create_download('foo', session)
+ self.mgr._do_downloads(session)
+ download = self.mgr.get_download(session, 'foo')
+ self.assertEqual(download.status, 'complete')
+
+ # No downloads left.
+ self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
+
+ @wrap_test_db
+ def test_max_attempts(self):
+ """
+ A Download will only be attempted so many times, it will eventually be deleted.
+ """
+ _, session = get_db_context()
+
+ http_downloader = HTTPDownloader()
+ http_downloader.do_download.return_value = False
+ self.mgr.register_downloader(http_downloader)
+
+ self.mgr.create_download('https://example.com', session)
+ download = session.query(Download).one()
+ self.assertEqual(download.attempts, 1)
+
+ self.mgr.create_download('https://example.com', session)
+ download = session.query(Download).one()
+ self.assertEqual(download.attempts, 2)
+
+ # There are no further attempts.
+ http_downloader.do_download.side_effect = UnrecoverableDownloadError()
+ self.mgr.create_download('https://example.com', session)
+ download = session.query(Download).one()
+ self.assertEqual(download.attempts, 3)
+ self.assertEqual(download.status, 'failed')
+
+ @wrap_test_db
+ def test_recurring_downloads(self):
+ """
+ A recurring Download should be downloaded repeatedly forever.
+ """
+ _, session = get_db_context()
+
+ http_downloader = HTTPDownloader()
+ http_downloader.do_download.return_value = True
+ http_downloader.do_download: MagicMock # noqa
+ self.mgr.register_downloader(http_downloader)
+
+ # Download every hour.
+ self.mgr.recurring_download('https://example.com', 3600, skip_download=True)
+
+ # One download is scheduled.
+ downloads = self.mgr.get_new_downloads(session)
+ self.assertEqual(
+ [(i.url, i.frequency) for i in downloads],
+ [('https://example.com', 3600)]
+ )
+
+ with mock.patch('wrolpi.downloader.now') as mock_now:
+ now = local_timezone(datetime(2020, 1, 1, 0, 0, 0))
+ mock_now.return_value = now
+
+ # Download is processed and successful, no longer "new".
+ self.mgr.do_downloads_sync()
+ http_downloader.do_download.assert_called_once()
+ self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
+ downloads = list(self.mgr.get_recurring_downloads(session))
+ self.assertEqual(len(downloads), 1)
+ download = downloads[0]
+ expected = local_timezone(datetime(2020, 1, 1, 1, 0, 0))
+ self.assertEqual(download.next_download, expected)
+ self.assertEqual(download.last_successful_download, now)
+
+ # Download is not due for an hour.
+ self.mgr.renew_recurring_downloads(session)
+ self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
+ self.assertEqual(download.last_successful_download, now)
+
+ # Download is due an hour later.
+ mock_now.return_value = local_timezone(datetime(2020, 1, 1, 1, 0, 0))
+ self.mgr.renew_recurring_downloads(session)
+ downloads = list(self.mgr.get_new_downloads(session))
+ self.assertEqual(len(downloads), 1)
+ download = downloads[0]
+ # Download is "new" but has not been done a second time.
+ self.assertEqual(download.next_download, expected)
+ self.assertEqual(download.last_successful_download, now)
+
+ # Try the download, but it fails.
+ http_downloader.do_download.reset_mock()
+ http_downloader.do_download.return_value = False
+ self.mgr.do_downloads_sync()
+ http_downloader.do_download.assert_called_once()
+ download = session.query(Download).one()
+ # Download is deferred, last successful download remains the same.
+ self.assertEqual(download.status, 'deferred')
+ self.assertEqual(download.last_successful_download, now)
+ # Download should be retried after the DEFAULT_RETRY_FREQUENCY.
+ self.assertEqual(download.next_download, expected + timedelta(hours=1))
+
+ # Try the download again, it finally succeeds.
+ http_downloader.do_download.reset_mock()
+ now = local_timezone(datetime(2020, 1, 1, 3, 0, 0))
+ mock_now.return_value = now
+ http_downloader.do_download.return_value = True
+ self.mgr.renew_recurring_downloads(session)
+ self.mgr.do_downloads_sync()
+ http_downloader.do_download.assert_called_once()
+ download = session.query(Download).one()
+ self.assertEqual(download.status, 'complete')
+ self.assertEqual(download.last_successful_download, now)
+ self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0)))
+
+ @wrap_test_db
+ @mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0)))
+ def test_delete_old_once_downloads(self):
+ """
+ Once-downloads over a month old should be deleted.
+ """
+ permissive_downloader = PermissiveDownloader(priority=0)
+ self.mgr.register_downloader(permissive_downloader)
+
+ _, session = get_db_context()
+ # Recurring downloads should not be deleted.
+ d1 = self.mgr.create_download('https://example.com/1', session, skip_download=True)
+ d2 = self.mgr.create_download('https://example.com/2', session, skip_download=True)
+ d1.frequency = 1
+ d2.frequency = 1
+ d2.started()
+ # Should be deleted.
+ d3 = self.mgr.create_download('https://example.com/3', session, skip_download=True)
+ d4 = self.mgr.create_download('https://example.com/4', session, skip_download=True)
+ d3.complete()
+ d4.complete()
+ d3.last_successful_download = local_timezone(datetime(2020, 1, 1, 0, 0, 0))
+ d4.last_successful_download = local_timezone(datetime(2020, 5, 1, 0, 0, 0))
+ # Not a month old.
+ d5 = self.mgr.create_download('https://example.com/5', session, skip_download=True)
+ d5.last_successful_download = local_timezone(datetime(2020, 6, 1, 0, 0, 0))
+ # An old, but pending download should not be deleted.
+ d6 = self.mgr.create_download('https://example.com/6', session, skip_download=True)
+ d6.last_successful_download = local_timezone(datetime(2020, 4, 1, 0, 0, 0))
+ d6.started()
+
+ self.mgr.delete_old_once_downloads()
+
+ # Two old downloads are deleted.
+ downloads = self.mgr._downloads_sorter(session.query(Download).all())
+ expected = [
+ dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'),
+ dict(url='https://example.com/6', attempts=1, status='pending'),
+ dict(url='https://example.com/1', frequency=1, attempts=0, status='new'),
+ dict(url='https://example.com/5', attempts=0, status='new'),
+ ]
+ for download, expected in zip_longest(downloads, expected):
+ self.assertDictContains(download.dict(), expected)
+
+ @wrap_test_db
+ def test_create_downloads(self):
+ """
+ Multiple downloads can be scheduled using DownloadManager.create_downloads.
+ """
+ http_downloader = HTTPDownloader(priority=0)
+ self.mgr.register_downloader(http_downloader)
+
+ _, session = get_db_context()
+
+ # Both URLs are valid and are scheduled.
+ self.mgr.create_downloads(['https://example.com/1', 'https://example.com/2'], skip_download=True)
+ downloads = self.mgr.get_downloads(session)
+ self.assertEqual({i.url for i in downloads}, {'https://example.com/1', 'https://example.com/2'})
+
+ # One URL is bad, neither should be scheduled.
+ self.assertRaises(InvalidDownload, self.mgr.create_downloads, ['https://example.com/3', 'bad url should fail'],
+ skip_download=True)
+ downloads = self.mgr.get_downloads(session)
+ self.assertEqual({i.url for i in downloads}, {'https://example.com/1', 'https://example.com/2'})
diff --git a/wrolpi/test/test_media_path.py b/wrolpi/test/test_media_path.py
index 1feda096..fd76b0a6 100644
--- a/wrolpi/test/test_media_path.py
+++ b/wrolpi/test/test_media_path.py
@@ -4,6 +4,7 @@ import tempfile
import unittest
from unittest import mock
+from wrolpi.common import set_test_media_directory
from wrolpi.media_path import MediaPath
from wrolpi.root_api import CustomJSONEncoder
@@ -12,11 +13,13 @@ class TestMediaPath(unittest.TestCase):
def setUp(self):
self.test_dir = tempfile.TemporaryDirectory()
+ set_test_media_directory(pathlib.Path(self.test_dir.name))
path = pathlib.Path(self.test_dir.name).absolute()
self.patch = mock.patch('wrolpi.media_path.get_media_directory', lambda: path)
self.patch.start()
def tearDown(self):
+ set_test_media_directory(None)
self.test_dir.cleanup()
self.patch.stop()
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
new file mode 100644
index 00000000..9346f31f
--- /dev/null
+++ b/wrolpi/test/test_root_api.py
@@ -0,0 +1,156 @@
+import json
+from http import HTTPStatus
+from itertools import zip_longest
+
+from wrolpi.dates import strptime
+from wrolpi.db import get_db_session
+from wrolpi.downloader import Download
+from wrolpi.root_api import api_app
+from wrolpi.test.common import TestAPI, wrap_test_db
+
+
+class TestRootAPI(TestAPI):
+
+ def test_index(self):
+ """
+ Index should have some details in an HTML response
+ """
+ request, response = api_app.test_client.get('/')
+ assert response.status_code == HTTPStatus.OK
+ assert b'html' in response.body
+
+ def test_echo(self):
+ """
+ Echo should send back a JSON object with our request details.
+ """
+ request, response = api_app.test_client.get('/api/echo?hello=world')
+ assert response.status_code == HTTPStatus.OK
+ self.assertDictContains(response.json['form'], {})
+ assert response.json['method'] == 'GET', 'Method was not GET'
+ assert response.json['json'] is None, 'JSON was not empty'
+ assert response.json['args'] == {'hello': ['world']}
+
+ data = {'foo': 'bar'}
+ request, response = api_app.test_client.post('/api/echo', data=json.dumps(data))
+ assert response.status_code == HTTPStatus.OK
+ self.assertDictContains(response.json['form'], {})
+ assert response.json['method'] == 'POST', 'Method was not POST'
+ assert response.json['json'] == data, 'JSON was not data'
+ assert response.json['args'] == {}
+
+ def test_valid_regex(self):
+ """
+ The endpoint should only return valid if the regex is valid.
+ """
+ data = {'regex': 'foo'}
+ request, response = api_app.test_client.post('/api/valid_regex', data=json.dumps(data))
+ assert response.status_code == HTTPStatus.OK
+ assert json.loads(response.body) == {'valid': True, 'regex': 'foo'}
+
+ data = {'regex': '.*(title match).*'}
+ request, response = api_app.test_client.post('/api/valid_regex', data=json.dumps(data))
+ assert response.status_code == HTTPStatus.OK
+ assert json.loads(response.body) == {'valid': True, 'regex': '.*(title match).*'}
+
+ data = {'regex': '.*(missing parenthesis.*'}
+ request, response = api_app.test_client.post('/api/valid_regex', data=json.dumps(data))
+ assert response.status_code == HTTPStatus.BAD_REQUEST
+ assert json.loads(response.body) == {'valid': False, 'regex': '.*(missing parenthesis.*'}
+
+ def test_get_settings(self):
+ request, response = api_app.test_client.get('/api/settings')
+ self.assertOK(response)
+
+ @wrap_test_db
+ def test_get_downloads(self):
+ request, response = api_app.test_client.get('/api/download')
+ self.assertDictContains(response.json, {'once_downloads': [], 'recurring_downloads': []})
+
+ with get_db_session(commit=True) as session:
+ d1 = Download(url='https://example.com/1', status='complete')
+ d2 = Download(url='https://example.com/2', status='pending')
+ d3 = Download(url='https://example.com/3', status='deferred')
+ d4 = Download(url='https://example.com/4', status='deferred')
+ d5 = Download(url='https://example.com/5', status='new')
+ d6 = Download(url='https://example.com/6', status='deferred', frequency=60)
+ d7 = Download(url='https://example.com/7', status='pending', frequency=60)
+ session.add_all([d1, d2, d3, d4, d5, d6, d7])
+
+ request, response = api_app.test_client.get('/api/download')
+ expected = [
+ {'url': 'https://example.com/2', 'status': 'pending'},
+ {'url': 'https://example.com/5', 'status': 'new'},
+ {'url': 'https://example.com/3', 'status': 'deferred'},
+ {'url': 'https://example.com/4', 'status': 'deferred'},
+ {'url': 'https://example.com/1', 'status': 'complete'},
+ ]
+ for download, expected in zip_longest(response.json['once_downloads'], expected):
+ self.assertDictContains(download, expected)
+ expected = [
+ {'url': 'https://example.com/7', 'status': 'pending'},
+ {'url': 'https://example.com/6', 'status': 'deferred'},
+ ]
+ for download, expected in zip_longest(response.json['recurring_downloads'], expected):
+ self.assertDictContains(download, expected)
+
+ @wrap_test_db
+ def test_downloads_sorter(self):
+ with get_db_session(commit=True) as session:
+ downloads = [
+ dict(status='complete', last_successful_download=strptime('2020-01-01 00:00:01')),
+ dict(status='complete', last_successful_download=strptime('2020-01-01 00:00:03')),
+ dict(status='complete', last_successful_download=strptime('2020-01-01 00:00:02')),
+ # last_successful_download are equal, so id is used next.
+ dict(status='pending', last_successful_download=strptime('2020-01-01 00:00:01'), url='1'),
+ dict(status='pending', last_successful_download=strptime('2020-01-01 00:00:01'), url='2'),
+
+ dict(status='pending', last_successful_download=strptime('2020-01-01 00:00:04')),
+ dict(status='failed'),
+ dict(status='failed', last_successful_download=strptime('2020-01-01 00:00:01')),
+ ]
+ for download in downloads:
+ session.add(Download(url=download.pop('url', 'https://example.com'), **download))
+
+ expected = [
+ dict(status='pending', last_successful_download=strptime('2020-01-01 00:00:04').timestamp()),
+ dict(status='pending', last_successful_download=strptime('2020-01-01 00:00:01').timestamp(), url='1'),
+ dict(status='pending', last_successful_download=strptime('2020-01-01 00:00:01').timestamp(), url='2'),
+ dict(status='failed'),
+ dict(status='failed', last_successful_download=strptime('2020-01-01 00:00:01').timestamp()),
+ dict(status='complete', last_successful_download=strptime('2020-01-01 00:00:03').timestamp()),
+ dict(status='complete', last_successful_download=strptime('2020-01-01 00:00:02').timestamp()),
+ dict(status='complete', last_successful_download=strptime('2020-01-01 00:00:01').timestamp()),
+ ]
+ request, response = api_app.test_client.get('/api/download')
+ once_downloads = response.json['once_downloads']
+ for d1, d2 in zip_longest(expected, once_downloads):
+ self.assertDictContains(d2, d1)
+
+ @wrap_test_db
+ def test_downloads_sorter_recurring(self):
+ with get_db_session(commit=True) as session:
+ downloads = [
+ dict(status='complete', frequency=1),
+ dict(status='complete', frequency=2, next_download=strptime('2020-01-01 00:00:01')),
+ dict(status='complete', frequency=2, next_download=strptime('2020-01-01 00:00:02')),
+ dict(status='pending', frequency=1),
+ dict(status='pending', frequency=4),
+ dict(status='failed', frequency=1),
+ dict(status='failed', frequency=1),
+ ]
+ for download in downloads:
+ session.add(Download(url='https://example.com', **download))
+
+ expected = [
+ dict(status='pending', frequency=1),
+ dict(status='pending', frequency=4),
+ dict(status='failed', frequency=1),
+ dict(status='failed'),
+ dict(status='complete', frequency=2, next_download=strptime('2020-01-01 00:00:01').timestamp()),
+ dict(status='complete', frequency=2, next_download=strptime('2020-01-01 00:00:02').timestamp()),
+ dict(status='complete', frequency=1),
+ ]
+ request, response = api_app.test_client.get('/api/download')
+ recurring_downloads = response.json['recurring_downloads']
+ for d1, d2 in zip_longest(expected, recurring_downloads):
+ self.assertDictContains(d2, d1)