@@ -110,9 +114,10 @@ export function VideoCard({video}) {
-
- {channel.name}
-
+ {channel &&
+
+ {channel.name}
+ }
{upload_date}
@@ -229,6 +234,27 @@ export function humanFileSize(bytes, si = false, dp = 1) {
return bytes.toFixed(dp) + ' ' + units[u];
}
+export function humanNumber(num, dp = 1) {
+ // Convert large numbers to a more human readable format.
+ // >> humanNumber(1000)
+ // 1.0k
+ // >> humanNumber(1500000)
+ // 1.5m
+ const divisor = 1000;
+ if (Math.abs(num) < divisor) {
+ return num;
+ }
+ const units = ['k', 'm', 'b'];
+ let i = -1;
+ const r = 10 ** dp;
+ do {
+ num /= divisor;
+ ++i;
+ } while (Math.round(Math.abs(num) * r) / r >= divisor && i < units.length - 1);
+
+ return num.toFixed(dp) + units[i];
+}
+
export class APIForm extends React.Component {
constructor(props) {
@@ -317,7 +343,7 @@ export class APIForm extends React.Component {
this.setState({
disabled: true,
error: false,
- errors: this.getEmptyErrors,
+ errors: this.getEmptyErrors(),
loading: true,
success: false,
})
diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js
new file mode 100644
index 00000000..de4356cd
--- /dev/null
+++ b/app/src/components/Upload.js
@@ -0,0 +1,74 @@
+import React from "react";
+import {Button, Form, Header, Segment, TextArea} from "semantic-ui-react";
+import {postDownload} from "../api";
+
+const validUrl = /^(http|https):\/\/[^ "]+$/;
+
+class Downloader extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ urls: '',
+ valid: true,
+ };
+ }
+
+ submitDownload = async () => {
+ if (this.state.urls) {
+ let response = await postDownload(this.state.urls);
+ if (response.status === 204) {
+ this.setState({urls: ''});
+ }
+ }
+ }
+
+ handleInputChange = async (event, {name, value}) => {
+ this.setState({[name]: value}, this.validateUrls);
+ }
+
+ validateUrls = async () => {
+ // Validate that all URLs in state are valid.
+ if (this.state.urls) {
+ let urls = this.state.urls.split(/\r?\n/);
+ let valid = true;
+ urls.forEach((url) => {
+ if (valid && url && !validUrl.test(url)) {
+ valid = false;
+ }
+ })
+ this.setState({valid});
+ }
+ }
+
+ render() {
+ let textareaPlaceholder = 'Enter one URL per line';
+ let disabled = !this.state.urls || !this.state.valid;
+
+ return (
+
+ )
+ }
+}
+
+export class Saver extends React.Component {
+
+ render() {
+ return (
+ <>
+
+
+
+ >
+ )
+ }
+}
diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js
index 152aebba..deb0e27c 100644
--- a/app/src/components/VideoPlayer.js
+++ b/app/src/components/VideoPlayer.js
@@ -3,7 +3,7 @@ import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {deleteVideo, favoriteVideo} from "../api";
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
import {Link} from "react-router-dom";
-import {humanFileSize, uploadDate, VideoCard} from "./Common";
+import {humanFileSize, humanNumber, uploadDate, VideoCard} from "./Common";
import {Confirm, Container, Segment, Tab} from "semantic-ui-react";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
@@ -24,15 +24,15 @@ function VideoPage(props) {
props.history.goBack();
}
- let videoUrl = `${MEDIA_PATH}/${channel.directory}/${encodeURIComponent(video.video_path)}`;
+ let videoUrl = `${MEDIA_PATH}/${encodeURIComponent(video.video_path)}`;
let posterUrl = null;
if (video.poster_path) {
- posterUrl = `${MEDIA_PATH}/${channel.directory}/${encodeURIComponent(video.poster_path)}`;
+ posterUrl = `${MEDIA_PATH}/${encodeURIComponent(video.poster_path)}`;
}
let captionsUrl = null;
if (video.caption_path) {
- captionsUrl = `${MEDIA_PATH}/${channel.directory}/${encodeURIComponent(video.caption_path)}`;
+ captionsUrl = `${MEDIA_PATH}/${encodeURIComponent(video.caption_path)}`;
}
let description = 'No description available.';
@@ -78,16 +78,19 @@ function VideoPage(props) {
let statisticsPane = {
menuItem: 'Statistics', render: () =>
Size
- {humanFileSize(video.size)}
+ {video.size ? humanFileSize(video.size) : '???'}
+
+ Source URL
+ {video.url || 'N/A'}
Source ID
- {video.source_id}
+ {video.source_id || 'N/A'}
View Count
- {viewCount}
+ {viewCount ? humanNumber(viewCount) : 'N/A'}
File Name
- {video.channel.directory}/{video.video_path}
+ {video.video_path}
}
@@ -117,9 +120,9 @@ function VideoPage(props) {
{video.title}
{video.upload_date && {uploadDate(video.upload_date)}
}
-
+ {channel &&
{channel.name}
-
+ }
diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js
index d539957f..282184ae 100644
--- a/app/src/components/Videos.js
+++ b/app/src/components/Videos.js
@@ -10,12 +10,11 @@ import Paginator, {
scrollToTop,
searchOrders,
secondsToString,
- uploadDate,
VideoCards,
videoOrders
} from "./Common"
import VideoPage from "./VideoPlayer";
-import {download, downloadHistory, getChannel, getStatistics, getVideo, refresh, searchVideos} from "../api";
+import {download, getChannel, getStatistics, getVideo, refresh, searchVideos} from "../api";
import {Button, Dropdown, Form, Grid, Header, Icon, Input, Loader, Segment, Statistic, Tab} from "semantic-ui-react";
import * as QueryString from 'query-string';
import Container from "semantic-ui-react/dist/commonjs/elements/Container";
@@ -28,15 +27,10 @@ class ManageVideos extends React.Component {
super(props);
this.state = {
streamUrl: null,
- downloadHistory: null,
mostRecentDownload: '',
}
}
- async componentDidMount() {
- await this.downloadHistory();
- }
-
download = async (e) => {
e.preventDefault();
let response = await download();
@@ -53,15 +47,6 @@ class ManageVideos extends React.Component {
}
}
- async downloadHistory() {
- let response = await downloadHistory();
- if (response && response.history.length > 0) {
- let history = response.history;
- let mostRecentDownload = uploadDate(history[0]['info_date']);
- this.setState({downloadHistory: history, mostRecentDownload: mostRecentDownload});
- }
- }
-
render() {
return (
@@ -75,7 +60,6 @@ class ManageVideos extends React.Component {
- Most recent download: {this.state.mostRecentDownload}
@@ -102,6 +86,7 @@ export class VideoWrapper extends React.Component {
prev: null,
next: null,
channel: null,
+ no_channel: null,
}
}
@@ -112,8 +97,13 @@ export class VideoWrapper extends React.Component {
async fetchVideo() {
// Get and display the Video specified in the Router match
let [video, prev, next] = await getVideo(this.props.match.params.video_id);
- let channel = await getChannel(this.props.match.params.channel_link);
- this.setState({video, prev, next, channel}, scrollToTop);
+ let channel_link = this.props.match.params.channel_link;
+ let channel = channel_link ? await getChannel(channel_link) : null;
+ let no_channel = false;
+ if (!channel) {
+ no_channel = true;
+ }
+ this.setState({video, prev, next, channel, no_channel}, scrollToTop);
}
async componentDidUpdate(prevProps, prevState) {
@@ -125,7 +115,7 @@ export class VideoWrapper extends React.Component {
}
render() {
- if (this.state.video && this.state.channel) {
+ if (this.state.video && (this.state.no_channel || this.state.channel)) {
return
} else {
return
diff --git a/conftest.py b/conftest.py
index d2e0ba61..bbcf7a76 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,6 @@
# Import conftest for test fixtures
-import os
-
+from wrolpi.common import import_modules
from wrolpi.conftest import * # noqa
-os.environ['DB_PORT'] = '54321'
+# Import modules and their conftest.py.
+import_modules()
diff --git a/docker-compose.yml b/docker-compose.yml
index 2472a21c..02df6191 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,7 +8,6 @@ services:
- POSTGRES_PASSWORD=wrolpi
ports:
- '54321:5432'
- - '5432:5432'
healthcheck:
test: [ 'CMD-SHELL', 'pg_isready -U postgres' ]
diff --git a/main.py b/main.py
index c6722b4e..73b70d6c 100755
--- a/main.py
+++ b/main.py
@@ -6,11 +6,13 @@ import logging
import sys
import pytz
+from sanic import Sanic
-from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS
-from wrolpi.common import logger, get_config
+from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, after_startup, limit_concurrent
+from wrolpi.common import logger, get_config, wrol_mode_enabled, import_modules
from wrolpi.dates import set_timezone
-from wrolpi.vars import PROJECT_DIR, MODULES_DIR
+from wrolpi.downloader import download_manager
+from wrolpi.vars import PROJECT_DIR
logger = logger.getChild('wrolpi-main')
@@ -40,7 +42,7 @@ def db_main(args):
return 0
-async def main():
+async def main(loop):
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='count')
@@ -70,18 +72,7 @@ async def main():
set_timezone(tz)
# Import the API in every module. Each API should attach itself to `root_api`.
- try:
- modules = [i.name for i in MODULES_DIR.iterdir() if i.is_dir() and not i.name.startswith('_')]
- for module in modules:
- module = f'modules.{module}.api'
- logger.debug(f'Importing {module}')
- __import__(module, globals(), locals(), [], 0)
- except ImportError as e:
- logger.fatal('No modules could be found!', exc_info=e)
- raise
-
- if not modules:
- raise Exception('No modules could be found!')
+ import_modules()
# Run the startup functions
for func in BEFORE_STARTUP_FUNCTIONS:
@@ -93,8 +84,8 @@ async def main():
except Exception as e:
logger.warning(f'Startup {func} failed!', exc_info=e)
- # Run the API
- return root_api.main(args)
+ # Run the API.
+ return root_api.main(loop, args)
async def set_log_level(args):
@@ -113,7 +104,32 @@ async def set_log_level(args):
logger.warning(f'Logging level: {level_name}')
+@after_startup
+@limit_concurrent(1)
+def periodic_downloads(app: Sanic, loop):
+ """
+ A simple function that perpetually calls downloader_manager.do_downloads() after sleeping.
+ """
+ if wrol_mode_enabled():
+ logger.warning(f'Not starting download worker because WROL Mode is enabled.')
+ return
+
+ # Set all downloads to new.
+ download_manager.reset_downloads()
+
+ async def _periodic_download():
+ download_manager.renew_recurring_downloads()
+ download_manager.delete_old_once_downloads()
+ await download_manager.do_downloads()
+ seconds = 60
+ logger.debug(f'Waiting {seconds} seconds to start next download')
+ await asyncio.sleep(seconds)
+ app.add_task(_periodic_download())
+
+ app.add_task(_periodic_download())
+
+
if __name__ == '__main__':
- loop = asyncio.get_event_loop()
- result = loop.run_until_complete(main())
+ loop_ = asyncio.get_event_loop()
+ result = loop_.run_until_complete(main(loop_))
sys.exit(result)
diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py
index 2d48cc88..eaaf2f00 100644
--- a/modules/archive/__init__.py
+++ b/modules/archive/__init__.py
@@ -1,6 +1,31 @@
+from abc import ABC
+
from wrolpi.common import logger
+from wrolpi.downloader import Downloader, Download
+from wrolpi.errors import UnrecoverableDownloadError
+from . import lib
from .api import bp # noqa
PRETTY_NAME = 'Archive'
logger = logger.getChild(__name__)
+
+
+class ArchiveDownloader(Downloader, ABC):
+ @staticmethod
+ def valid_url(url: str) -> bool:
+ """
+ Archiver will attempt to archive anything, so it should be last!
+ """
+ return True
+
+ def do_download(self, download: Download):
+ if download.attempts > 3:
+ raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
+
+ lib.new_archive(download.url, sync=True)
+ return True
+
+
+# Archive downloader is the last downloader which should be used.
+archive_downloader = ArchiveDownloader(priority=100)
diff --git a/modules/archive/api.py b/modules/archive/api.py
index bd6d7f86..e7b35a9f 100644
--- a/modules/archive/api.py
+++ b/modules/archive/api.py
@@ -64,7 +64,7 @@ async def search_archives(_: Request, data: dict):
return json_response(ret)
-@bp.post(':refresh')
+@bp.post('/refresh')
@wrol_mode_check
async def refresh_archives(_: Request):
asyncio.ensure_future(lib.refresh_archives())
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 326212e1..8c4bbc11 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -6,17 +6,17 @@ import pathlib
import re
from itertools import groupby
from typing import Iterator
-from urllib.parse import urlparse
import requests
from bs4 import BeautifulSoup
from sqlalchemy.orm import Session
from modules.archive.models import URL, Domain, Archive
-from wrolpi.common import get_media_directory, logger, chunks
+from wrolpi.common import get_media_directory, logger, chunks, extract_domain
from wrolpi.dates import now, strptime_ms, strftime_ms
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import InvalidDomain, UnknownURL, PendingArchive, InvalidArchive
+from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@@ -27,21 +27,6 @@ def get_archive_directory() -> pathlib.Path:
return get_media_directory() / 'archive'
-def extract_domain(url):
- """
- Extract the domain from a URL. Remove leading www.
-
- >>> extract_domain('https://www.example.com/foo')
- 'example.com'
- """
- parsed = urlparse(url)
- domain = parsed.netloc
- if domain.startswith('www.'):
- # Remove leading www.
- domain = domain[4:]
- return domain
-
-
def get_domain_directory(url: str) -> pathlib.Path:
"""
Get the archive directory for a particular domain.
@@ -161,7 +146,7 @@ def new_archive(url: str, sync: bool = False):
url_.latest_id = archive_id
url_.latest_datetime = now()
- if sync:
+ if PYTEST or sync:
return _do_archive(url, archive_id)
else:
# Run the real archive process in the future.
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index ff68f8c0..d0a779fb 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -312,7 +312,7 @@ class TestArchive(TestAPI):
archive: Archive = session.query(Archive).one()
self.assertEqual(url.latest, archive)
# latest_datetime is set, the timestamp in the file name is assumed to be UTC.
- self.assertEqual(str(url.latest_datetime), '2021-10-05 22:20:10.346823-06:00')
+ self.assertEqual(str(url.latest_datetime), '2021-10-05 16:20:10.346823-06:00')
# Running the refresh does not result in a new archive.
_refresh_archives()
diff --git a/modules/inventory/models.py b/modules/inventory/models.py
index 0b285bc6..64d4c365 100644
--- a/modules/inventory/models.py
+++ b/modules/inventory/models.py
@@ -27,8 +27,8 @@ class Item(Base, ModelHelper):
inventory = relationship('Inventory', primaryjoin="Item.inventory_id==Inventory.id", back_populates='items')
def __repr__(self):
- return f'- '
+ return f'
- '
class Inventory(Base, ModelHelper):
@@ -43,7 +43,7 @@ class Inventory(Base, ModelHelper):
items: InstrumentedList = relationship('Item', foreign_keys='Item.inventory_id')
def __repr__(self):
- return f''
+ return f''
def dict(self):
d = super().dict()
diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py
index e69de29b..8bfe14d6 100644
--- a/modules/videos/__init__.py
+++ b/modules/videos/__init__.py
@@ -0,0 +1 @@
+from modules.videos import downloader
diff --git a/modules/videos/api.py b/modules/videos/api.py
index 212c0f73..dd629c39 100644
--- a/modules/videos/api.py
+++ b/modules/videos/api.py
@@ -1,28 +1,25 @@
import asyncio
-import random
from functools import wraps
from http import HTTPStatus
-from multiprocessing import Event
-import requests
from sanic import Blueprint, response
from sanic.request import Request
from wrolpi import before_startup
from wrolpi.common import create_websocket_feed, get_sanic_url, \
- wrol_mode_check, ProgressReporter, get_config
+ wrol_mode_check
from wrolpi.common import logger
from wrolpi.root_api import add_blueprint, json_response
from wrolpi.schema import validate_doc, JSONErrorResponse
from .channel.api import channel_bp
-from .downloader import update_channels, download_all_missing_videos
-from .lib import process_video_meta_data, _refresh_videos, get_statistics, get_download_history
+from .channel.lib import download_channel, spread_channel_downloads
+from .lib import _refresh_videos, get_statistics
from .schema import StreamResponse, \
- FavoriteRequest, FavoriteResponse, VideosStatisticsResponse, DownloadHistoryResponse
+ FavoriteRequest, FavoriteResponse, VideosStatisticsResponse
from .video.api import video_bp
from .video.lib import set_video_favorite
-content_bp = Blueprint('Video Content', '/api/videos')
+content_bp = Blueprint('VideoContent', '/api/videos')
bp = Blueprint('Videos', '/api/videos').group(
content_bp, # view and manage video content and settings
channel_bp, # view and manage channels
@@ -35,8 +32,8 @@ logger = logger.getChild(__name__)
refresh_queue, refresh_event = create_websocket_feed('refresh', '/feeds/refresh', content_bp)
-@content_bp.post(':refresh')
-@content_bp.post(':refresh/')
+@content_bp.post('/refresh')
+@content_bp.post('/refresh/')
@validate_doc(
summary='Search for videos that have previously been downloaded and stored.',
produces=StreamResponse,
@@ -63,8 +60,7 @@ async def refresh(_, link: str = None):
await refresh_videos(channel_links)
refresh_logger.info('refresh complete')
- except Exception as e:
- refresh_queue.put({'error': 'Refresh failed. See server logs.', 'message': str(e)})
+ except Exception:
raise
finally:
refresh_event.clear()
@@ -78,8 +74,8 @@ async def refresh(_, link: str = None):
download_queue, download_event = create_websocket_feed('download', '/feeds/download', content_bp)
-@content_bp.post(':download')
-@content_bp.post(':download/')
+@content_bp.post('/download')
+@content_bp.post('/download/')
@validate_doc(
summary='Update channel catalogs, download any missing videos',
produces=StreamResponse,
@@ -88,7 +84,7 @@ download_queue, download_event = create_websocket_feed('download', '/feeds/downl
],
)
@wrol_mode_check
-async def download(_, link: str = None):
+def download(_, link: str = None):
download_logger = logger.getChild('download')
stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/download')
@@ -98,44 +94,18 @@ async def download(_, link: str = None):
if refresh_event.is_set():
return response.json({'error': 'Refresh is running. Cannot download.'})
- download_event.set()
+ download_channel(link)
- async def do_download():
- reporter = ProgressReporter(download_queue, 2)
- reporter.set_progress_total(0, 1)
- reporter.send_progress(0, 1, 'Download started')
-
- try:
- update_channels(reporter, link)
- download_logger.info('Updated all channel catalogs')
- download_all_missing_videos(reporter, link)
- reporter.finish(1, 'All videos have been downloaded')
-
- # Fill in any missing data for all videos.
- reporter.message(0, 'Processing and cleaning files')
- process_video_meta_data()
- reporter.finish(0, 'Processing and cleaning complete')
-
- download_logger.info('download complete')
- except Exception as e:
- logger.fatal(f'Download failed: {e}')
- reporter.error(0, 'Download failed. See server logs.')
- raise
- finally:
- download_event.clear()
-
- coro = do_download()
- asyncio.ensure_future(coro)
download_logger.debug('do_download scheduled')
return response.json({'code': 'stream-started', 'stream_url': stream_url})
@wraps(_refresh_videos)
async def refresh_videos(channel_links: list = None):
- return _refresh_videos(refresh_queue, channel_links=channel_links)
+ return _refresh_videos(channel_links=channel_links)
-@content_bp.post(':favorite')
+@content_bp.post('/favorite')
@validate_doc(
summary='Toggle the favorite flag on a video',
consumes=FavoriteRequest,
@@ -160,56 +130,6 @@ async def statistics(_: Request):
return json_response(ret, HTTPStatus.OK)
-@content_bp.get('/download_history')
-@validate_doc(
- summary='Retrieve video download history',
- produces=DownloadHistoryResponse,
-)
-async def download_history(_: Request):
- history = await get_download_history()
- ret = dict(history=history)
- return json_response(ret, HTTPStatus.OK)
-
-
-MIN_DOWNLOAD_FREQUENCY = 60 * 60 * 4 # 4 hours
-MAX_DOWNLOAD_FREQUENCY = 60 * 60 * 12 # 12 hours
-PERIODIC_DOWNLOAD_EVENT = Event()
-
-
-async def _periodic_download(min_download_frequency: int, max_download_frequency: int):
- """
- Wait some amount of time, download, then schedule the next download.
- """
- sleep_seconds = random.randint(min_download_frequency, max_download_frequency)
- logger.debug(f'Waiting {sleep_seconds} seconds before next periodic download')
- await asyncio.sleep(sleep_seconds)
-
- url = get_sanic_url(path='/api/videos:download')
- resp = requests.post(url)
- if resp.status_code != HTTPStatus.OK and resp.status_code != HTTPStatus.CONFLICT:
- # Download failed and wasn't already running.
- logger.warning(f'Periodic download failed with status_code={resp.status_code}')
- logger.warning(f'Periodic download response={resp}')
-
- # Schedule the next download
- asyncio.ensure_future(_periodic_download(min_download_frequency, max_download_frequency))
-
-
@before_startup
-def start_periodic_download():
- """
- Periodically download the videos.
- """
- if not PERIODIC_DOWNLOAD_EVENT.is_set():
- PERIODIC_DOWNLOAD_EVENT.set()
- logger.info(f'Starting periodic download')
-
- config = get_config()
- try:
- min_download_frequency = int(config.get('min_download_frequency', MIN_DOWNLOAD_FREQUENCY))
- max_download_frequency = int(config.get('max_download_frequency', MAX_DOWNLOAD_FREQUENCY))
- except Exception:
- logger.fatal(f'Failed to get min/max download frequencies from config!', exc_info=True)
- raise
-
- asyncio.ensure_future(_periodic_download(min_download_frequency, max_download_frequency))
+def startup_spread_channel_downloads():
+ spread_channel_downloads()
diff --git a/modules/videos/captions.py b/modules/videos/captions.py
index bd18d93a..f44d8235 100755
--- a/modules/videos/captions.py
+++ b/modules/videos/captions.py
@@ -1,4 +1,5 @@
#! /usr/bin/env python3
+import pathlib
from pathlib import Path
from typing import Generator, List, Union
@@ -7,8 +8,7 @@ import webvtt
from wrolpi.common import logger, chunks
from wrolpi.db import get_db_session
-from wrolpi.errors import UnknownCaptionFile
-from .common import get_absolute_video_caption
+from wrolpi.media_path import MediaPath
from .models import Video
@@ -43,9 +43,11 @@ def process_captions(video: Video):
"""
Parse and insert captions for a video record.
"""
- caption_path = get_absolute_video_caption(video)
- if not caption_path.exists():
- raise UnknownCaptionFile()
+ if isinstance(video.caption_path, MediaPath):
+ caption_path = video.caption_path.path
+ else:
+ caption_path = pathlib.Path(video.caption_path)
+
try:
lines = get_unique_caption_lines(str(caption_path))
block = '\n'.join(lines)
diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py
index bc0ce555..48eb5333 100644
--- a/modules/videos/channel/api.py
+++ b/modules/videos/channel/api.py
@@ -5,8 +5,8 @@ from sanic import response, Blueprint
from sanic.request import Request
from wrolpi.common import logger, wrol_mode_check, get_relative_to_media_directory, make_media_directory
-from wrolpi.root_api import json_response
from wrolpi.errors import UnknownDirectory
+from wrolpi.root_api import json_response
from wrolpi.schema import validate_doc, JSONErrorResponse
from .lib import get_minimal_channels, delete_channel, update_channel, get_channel, create_channel
from ..schema import ChannelsResponse, ChannelResponse, ChannelPostRequest, \
@@ -27,7 +27,7 @@ async def get_channels(_: Request):
return json_response({'channels': channels})
-@channel_bp.route('/', methods=['GET', 'OPTIONS'])
+@channel_bp.route('/', methods=['GET', 'OPTIONS'])
@validate_doc(
summary='Get a Channel',
produces=ChannelResponse,
@@ -36,7 +36,7 @@ async def get_channels(_: Request):
),
)
def channel_get(_: Request, link: str):
- channel = get_channel(link)
+ channel = get_channel(link=link)
channel.pop('info_json')
return json_response({'channel': channel})
@@ -61,6 +61,15 @@ def channel_post(_: Request, data: dict):
else:
raise
+ if download_frequency := data.get('download_frequency'):
+ try:
+ download_frequency = int(download_frequency)
+ except ValueError:
+ if download_frequency in ('null', 'None'):
+ download_frequency = None
+
+ data['download_frequency'] = download_frequency
+
channel = create_channel(data)
# Refresh the videos asynchronously
@@ -72,12 +81,11 @@ def channel_post(_: Request, data: dict):
{'Location': f'/api/videos/channels/{channel["link"]}'})
-@channel_bp.put('/')
-@channel_bp.patch('/')
+@channel_bp.put('/')
+@channel_bp.patch('/')
@validate_doc(
summary='Update a Channel',
consumes=ChannelPutRequest,
- produces=SuccessResponse,
responses=(
(HTTPStatus.NOT_FOUND, JSONErrorResponse),
(HTTPStatus.BAD_REQUEST, JSONErrorResponse),
@@ -90,7 +98,7 @@ def channel_update(_: Request, link: str, data: dict):
headers={'Location': f'/api/videos/channels/{channel.link}'})
-@channel_bp.delete('/')
+@channel_bp.delete('/')
@validate_doc(
summary='Delete a Channel',
produces=SuccessResponse,
diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index 51913e1d..d34d47d8 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -1,16 +1,19 @@
-from typing import List, Dict
+from collections import defaultdict
+from datetime import timedelta
+from typing import List, Dict, Union
from sqlalchemy.orm.exc import NoResultFound
-from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory
+from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory, logger
+from wrolpi.dates import today
from wrolpi.db import get_db_session, get_db_curs
+from wrolpi.downloader import download_manager, Download
from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError
from ..common import check_for_channel_conflicts
from ..lib import save_channels_config
from ..models import Channel
-DEFAULT_DOWNLOAD_FREQUENCY = 60 * 60 * 24 * 7 # weekly
-DEFAULT_DOWNLOAD_TIMEOUT = 60.0 * 10.0 # Ten minutes
+logger = logger.getChild(__name__)
async def get_minimal_channels() -> List[dict]:
@@ -21,7 +24,7 @@ async def get_minimal_channels() -> List[dict]:
# Get all channels, even if they don't have videos.
stmt = '''
SELECT
- c.id, name, link, directory, url, download_frequency, info_date, next_download
+ c.id, name, link, directory, url, download_frequency, info_date
FROM
channel AS c
ORDER BY LOWER(name)
@@ -64,15 +67,69 @@ async def get_channels_video_count() -> Dict[int, int]:
return video_counts
-@run_after(save_channels_config)
-def delete_channel(link):
- with get_db_session(commit=True) as session:
- try:
- channel = session.query(Channel).filter_by(link=link).one()
- except NoResultFound:
- raise UnknownChannel()
+def get_channel(source_id: str = None, link: str = None, url: str = None, return_dict: bool = True
+ ) -> Union[dict, Channel]:
+ """
+ Attempt to find a Channel using the provided params. The params are in order of reliability.
- channel.delete_with_videos()
+ Raises UnknownChannel if no channel is found.
+ """
+ with get_db_session() as session:
+ # Try by source_id first.
+ channel = None
+ if source_id:
+ channel = session.query(Channel).filter_by(source_id=source_id).one_or_none()
+ if not channel and link:
+ channel = session.query(Channel).filter_by(link=link).one_or_none()
+ if not channel and url:
+ channel = session.query(Channel).filter_by(url=url).one_or_none()
+
+ if not channel:
+ raise UnknownChannel(f'No channel matches {link=} {source_id=} {url=}')
+
+ logger.debug(f'Found {channel=} using {source_id=} {link=} {url=}')
+ return channel.dict() if return_dict else channel
+
+
+def _spread_by_frequency(channels: List[Channel]):
+ channels_by_frequency = defaultdict(lambda: [])
+ for channel in channels:
+ channels_by_frequency[channel.download_frequency].append(channel)
+
+ for frequency, channels in channels_by_frequency.items():
+ # The seconds between each download.
+ chunk = frequency // len(channels)
+ for channel in channels:
+ # My position in the channel group.
+ index = channels.index(channel)
+ # My next download will be distributed by my frequency and my position.
+ position = chunk * index
+ next_download = today() + timedelta(seconds=frequency + position)
+ yield dict(url=channel.url, frequency=frequency, next_download=next_download)
+
+
+def spread_channel_downloads():
+ """
+ Channels should be downloaded in a manner that is spread out over their frequency. For example, three channels
+ with a frequency of a week should be downloaded on different days that week.
+ """
+ with get_db_session(commit=True) as session:
+ channels = list(session.query(Channel).filter(
+ Channel.url != None,
+ Channel.url != '',
+ Channel.download_frequency != None
+ ).order_by(Channel.link).all()) # noqa
+
+ url_next_download = _spread_by_frequency(channels)
+
+ for info in url_next_download:
+ download = download_manager.get_download(session, url=info['url'])
+ if not download:
+ download = Download(url=info['url'])
+ session.add(download)
+
+ download.frequency = info['frequency']
+ download.next_download = info['next_download']
@run_after(save_channels_config)
@@ -101,7 +158,10 @@ def update_channel(data, link):
try:
data['download_frequency'] = int(data['download_frequency'])
except ValueError:
- raise APIError('Invalid download frequency')
+ if data['download_frequency'] in ('null', 'None', ''):
+ data['download_frequency'] = None
+ else:
+ raise APIError(f'Invalid download frequency {data["download_frequency"]}')
if data.get('match_regex') in ('None', 'null'):
data['match_regex'] = None
@@ -119,35 +179,28 @@ def update_channel(data, link):
# Apply the changes now that we've OK'd them
channel.update(data)
+ spread_channel_downloads()
+
return channel
-def get_channel(link) -> dict:
- """
- Get a Channel by it's `link`. Raise UnknownChannel if it does not exist.
- """
- with get_db_session() as session:
- try:
- channel = session.query(Channel).filter_by(link=link).one()
- except NoResultFound:
- raise UnknownChannel()
- return channel.dict()
-
-
@run_after(save_channels_config)
-def create_channel(data: dict) -> dict:
+@run_after(spread_channel_downloads)
+def create_channel(data: dict, return_dict: bool = True) -> Union[Channel, dict]:
"""
Create a new Channel. Check for conflicts with existing Channels.
"""
with get_db_session(commit=True) as session:
+ link = sanitize_link(data['name'])
try:
# Verify that the URL/Name/Link aren't taken
check_for_channel_conflicts(
session,
url=data.get('url'),
name=data['name'],
- link=sanitize_link(data['name']),
+ link=link,
directory=str(data['directory']),
+ source_id=data.get('source_id'),
)
except APIError as e:
raise ValidationError() from e
@@ -156,13 +209,34 @@ def create_channel(data: dict) -> dict:
name=data['name'],
url=data.get('url'),
match_regex=data.get('match_regex'),
- link=sanitize_link(data['name']),
+ link=link,
directory=str(data['directory']),
- download_frequency=data.get('download_frequency', DEFAULT_DOWNLOAD_FREQUENCY),
+ download_frequency=data.get('download_frequency'),
+ source_id=data.get('source_id'),
)
session.add(channel)
session.commit()
session.flush()
session.refresh(channel)
- return channel.dict()
+ if return_dict:
+ return channel.dict()
+ else:
+ return channel
+
+
+@run_after(save_channels_config)
+def delete_channel(link):
+ with get_db_session(commit=True) as session:
+ try:
+ channel = session.query(Channel).filter_by(link=link).one()
+ except NoResultFound:
+ raise UnknownChannel()
+
+ channel.delete_with_videos()
+
+
+def download_channel(link: str):
+ channel = get_channel(link=link, return_dict=False)
+ with get_db_session(commit=True) as session:
+ download_manager.create_download(channel.url, session)
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index aea45baf..656e2cc7 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -1,4 +1,5 @@
import json
+import os
import pathlib
import tempfile
from datetime import timedelta
@@ -9,14 +10,16 @@ import mock
import pytest
from modules.videos.api import refresh_queue
+from modules.videos.channel.lib import spread_channel_downloads, delete_channel
from modules.videos.lib import upsert_video
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.dates import now
from wrolpi.db import get_db_session
+from wrolpi.downloader import download_manager, Download
from wrolpi.errors import UnknownFile
from wrolpi.root_api import api_app
-from wrolpi.test.common import wrap_test_db, get_all_messages_in_queue, TestAPI
+from wrolpi.test.common import wrap_test_db, TestAPI
class TestVideoAPI(TestAPI):
@@ -39,83 +42,83 @@ class TestVideoAPI(TestAPI):
@wrap_test_db
def test_channel(self):
- with tempfile.TemporaryDirectory() as media_dir, \
- mock.patch('wrolpi.common.get_media_directory', lambda: pathlib.Path(media_dir)):
- channel_directory = tempfile.TemporaryDirectory(dir=media_dir).name
- pathlib.Path(channel_directory).mkdir()
- new_channel = dict(
- directory=channel_directory,
- match_regex='asdf',
- name='Example Channel 1',
- url='https://example.com/channel1',
- )
+ channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name
+ channel_directory_name = channel_directory.split('/')[-1]
+ pathlib.Path(channel_directory).mkdir()
- # Channel doesn't exist
- request, response = api_app.test_client.get('/api/videos/channels/examplechannel1')
- assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
+ new_channel = dict(
+ directory=channel_directory,
+ match_regex='asdf',
+ name='Example Channel 1',
+ url='https://example.com/channel1',
+ )
- # Create it
- request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
- assert response.status_code == HTTPStatus.CREATED
- location = response.headers['Location']
- request, response = api_app.test_client.get(location)
- created = response.json['channel']
- self.assertIsNotNone(created)
- self.assertIsNotNone(created['id'])
+ # Channel doesn't exist
+ request, response = api_app.test_client.get('/api/videos/channels/examplechannel1')
+ assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
- # Channel directory should be relative to the media directory
- assert not pathlib.Path(created['directory']).is_absolute(), \
- f'Channel directory is absolute: {created["directory"]}'
+ # Create it
+ request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.CREATED
+ location = response.headers['Location']
+ request, response = api_app.test_client.get(location)
+ created = response.json['channel']
+ self.assertIsNotNone(created)
+ self.assertIsNotNone(created['id'])
- # Get the link that was computed
- new_channel['link'] = response.json['channel']['link']
- assert new_channel['link']
+ # Channel directory should be relative to the media directory
+ assert not pathlib.Path(created['directory']).is_absolute(), \
+ f'Channel directory is absolute: {created["directory"]}'
- # Can't create it again
- request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
- assert response.status_code == HTTPStatus.BAD_REQUEST
+ # Get the link that was computed
+ new_channel['link'] = response.json['channel']['link']
+ assert new_channel['link']
- # Update it
- new_channel['name'] = 'Example Channel 2'
- new_channel['directory'] = str(new_channel['directory'])
- request, response = api_app.test_client.put(location, data=json.dumps(new_channel))
- assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
- request, response = api_app.test_client.get(location)
- assert response.status_code == HTTPStatus.OK
- self.assertDictContains(response.json['channel'], {
- 'id': 1,
- 'name': 'Example Channel 2',
- 'directory': channel_directory,
- 'match_regex': 'asdf',
- 'url': 'https://example.com/channel1',
- })
+ # Can't create it again
+ request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.BAD_REQUEST
- # Patch it
- patch = {'name': 'new name'}
- request, response = api_app.test_client.patch(location, data=json.dumps(patch))
- assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
- request, response = api_app.test_client.get(location)
- assert response.status_code == HTTPStatus.OK
- self.assertDictContains(response.json['channel'], {
- 'id': 1,
- 'name': 'new name',
- 'directory': channel_directory,
- 'match_regex': 'asdf',
- 'url': 'https://example.com/channel1',
- })
+ # Update it
+ new_channel['name'] = 'Example Channel 2'
+ new_channel['directory'] = str(new_channel['directory'])
+ request, response = api_app.test_client.put(location, data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
+ request, response = api_app.test_client.get(location)
+ assert response.status_code == HTTPStatus.OK
+ self.assertDictContains(response.json['channel'], {
+ 'id': 1,
+ 'name': 'Example Channel 2',
+ 'directory': channel_directory_name,
+ 'match_regex': 'asdf',
+ 'url': 'https://example.com/channel1',
+ })
- # Can't update channel that doesn't exist
- request, response = api_app.test_client.put('/api/videos/channels/doesnt_exist',
- data=json.dumps(new_channel))
- assert response.status_code == HTTPStatus.NOT_FOUND
+ # Patch it
+ patch = {'name': 'new name'}
+ request, response = api_app.test_client.patch(location, data=json.dumps(patch))
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
+ request, response = api_app.test_client.get(location)
+ assert response.status_code == HTTPStatus.OK
+ self.assertDictContains(response.json['channel'], {
+ 'id': 1,
+ 'name': 'new name',
+ 'directory': channel_directory_name,
+ 'match_regex': 'asdf',
+ 'url': 'https://example.com/channel1',
+ })
- # Delete the new channel
- request, response = api_app.test_client.delete(location)
- assert response.status_code == HTTPStatus.NO_CONTENT
+ # Can't update channel that doesn't exist
+ request, response = api_app.test_client.put('/api/videos/channels/doesnt_exist',
+ data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.NOT_FOUND
- # Cant delete it again
- request, response = api_app.test_client.delete(location)
- assert response.status_code == HTTPStatus.NOT_FOUND
+ # Delete the new channel
+ request, response = api_app.test_client.delete(location)
+ assert response.status_code == HTTPStatus.NO_CONTENT
+
+ # Cant delete it again
+ request, response = api_app.test_client.delete(location)
+ assert response.status_code == HTTPStatus.NOT_FOUND
@wrap_test_db
def test_channel_conflicts(self):
@@ -210,15 +213,61 @@ class TestVideoAPI(TestAPI):
assert response.status_code == HTTPStatus.CREATED, response.json
assert location != response.headers['Location']
+ @wrap_test_db
+ def test_refresh_no_channel(self):
+ with get_db_session() as session:
+ videos_path = pathlib.Path(self.tmp_dir.name) / 'videos'
+ videos_path.mkdir()
+
+ # A regular user-defined channel
+ channel1_path = videos_path / 'channel1'
+ channel1_path.mkdir()
+ vid1 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
+ vid1.touch()
+ vid2 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
+ vid2.touch()
+
+ channel1 = Channel(name='channel1', link='channel1', directory=channel1_path)
+ session.add(channel1)
+ session.commit()
+
+ # The special NO CHANNEL directory.
+ from modules.videos.downloader import get_no_channel_directory
+ no_channel_path = get_no_channel_directory()
+ no_channel_path.mkdir()
+ vid3 = no_channel_path / 'some video.mp4'
+ vid3.touch()
+ no_channel_subdir = no_channel_path / 'subdir'
+ no_channel_subdir.mkdir()
+ vid4 = no_channel_subdir / 'other video.mp4'
+ vid4.touch()
+
+ api_app.test_client.post('/api/videos/refresh')
+
+ videos = session.query(Video).order_by(Video.title).all()
+ self.assertEqual(len(videos), 4)
+
+ # Add some fake videos, they should be removed.
+ session.add(Video(title='vid5'))
+ session.add(Video(title='vid6', channel_id=channel1.id))
+ session.commit()
+
+ videos = session.query(Video).order_by(Video.title).all()
+ self.assertEqual(len(videos), 6)
+
+ api_app.test_client.post('/api/videos/refresh')
+
+ videos = session.query(Video).order_by(Video.title).all()
+ self.assertEqual(len(videos), 4)
+
@wrap_test_db
def test_refresh_videos(self):
# There should be no messages until a refresh is called.
pytest.raises(Empty, refresh_queue.get_nowait)
# Setup a fake channel directory.
- with get_db_session() as session, \
- tempfile.TemporaryDirectory() as channel_dir:
- channel_path = pathlib.Path(channel_dir)
+ with get_db_session() as session:
+ channel_path = pathlib.Path(self.tmp_dir.name)
# Files in subdirectories should be found and handled properly.
subdir = channel_path / 'subdir'
@@ -243,26 +292,26 @@ class TestVideoAPI(TestAPI):
poster2.touch()
# Create a channel, associate videos with it.
- channel = Channel(directory=channel_dir, link='foo', name='foo')
+ channel = Channel(directory=channel_path, link='foo', name='foo')
session.add(channel)
session.flush()
session.refresh(channel)
video1 = upsert_video(session, vid1, channel)
video2 = upsert_video(session, vid2, channel)
session.commit()
- self.assertEqual({i.video_path for i in channel.videos},
- {pathlib.Path('subdir/' + vid1.name), pathlib.Path(vid2.name)})
+ self.assertEqual({str(i.video_path.path) for i in channel.videos},
+ {f'{channel_path}/subdir/{vid1.name}', f'{channel_path}/{vid2.name}'})
# Poster files were found.
- self.assertEqual(video1.poster_path, pathlib.Path('subdir/' + poster1.name))
- self.assertEqual(video2.poster_path, pathlib.Path(poster2.name))
+ self.assertEqual(video1.poster_path.path, subdir / poster1.name)
+ self.assertEqual(video2.poster_path.path, channel_path / poster2.name)
# Add a bogus file, this should be removed during the refresh
- self.assertNotIn('foo', {i.video_path for i in channel.videos})
+ self.assertNotIn('foo', {i.video_path.path for i in channel.videos})
session.add(Video(video_path='foo', channel_id=channel.id))
session.flush()
session.refresh(channel)
- self.assertIn('foo', {str(i.video_path) for i in channel.videos})
+ self.assertIn(f'{self.tmp_dir.name}/foo', {str(i.video_path.path) for i in channel.videos})
self.assertEqual(len(channel.videos), 3)
# Add a video that isn't in the DB, it should be found and any meta files associated with it
@@ -277,10 +326,10 @@ class TestVideoAPI(TestAPI):
# Finally, call the refresh. Again, it should remove the "foo" video, then discover this 3rd video
# file and it's description.
- api_app.test_client.post('/api/videos:refresh')
+ api_app.test_client.post('/api/videos/refresh')
# Bogus file was removed
- self.assertNotIn('foo', {i.video_path for i in channel.videos})
+ self.assertNotIn(f'{self.tmp_dir.name}/foo', {i.video_path.path for i in channel.videos})
# Final channel video list we built
expected = {
@@ -290,7 +339,7 @@ class TestVideoAPI(TestAPI):
}
def str_or_none(i):
- return str(i) if i else None
+ return str(i.__json__()) if i else None
self.assertEqual(
{(str_or_none(i.video_path), str_or_none(i.poster_path), str_or_none(i.description_path))
@@ -300,10 +349,6 @@ class TestVideoAPI(TestAPI):
assert poster3.is_file(), 'Orphan jpg file was deleted!'
- # During the refresh process, messages are pushed to a queue, make sure there are messages there
- messages = get_all_messages_in_queue(refresh_queue)
- assert 'refresh-started' in [i.get('code') for i in messages]
-
@wrap_test_db
def test_get_channel_videos(self):
with get_db_session(commit=True) as session:
@@ -466,3 +511,93 @@ class TestVideoAPI(TestAPI):
response = do_search('b', 2)
assert [i['id'] for i in response.json['videos']] == [1, 2]
assert response.json['totals']['videos'] == 4
+
+ @wrap_test_db
+ def test_channel_no_download_frequency(self):
+ """
+ A channel does not require a download frequency.
+ """
+ channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name
+ os.mkdir(channel_directory)
+
+ new_channel = dict(
+ directory=channel_directory,
+ name='Example Channel 1',
+ url='https://example.com/channel1',
+ download_frequency=None,
+ )
+
+ # Create the Channel
+ request, response = api_app.test_client.post('/api/videos/channels', data=json.dumps(new_channel))
+ assert response.status_code == HTTPStatus.CREATED
+ location = response.headers['Location']
+
+ # No downloads are scheduled.
+ spread_channel_downloads()
+ with get_db_session() as session:
+ downloads = list(download_manager.get_new_downloads(session))
+ self.assertEqual(len(downloads), 0)
+
+ # Get the Channel
+ request, response = api_app.test_client.get(location)
+ channel = response.json['channel']
+ self.assertEqual(channel['download_frequency'], None)
+
+ # Update the Channel with a frequency.
+ new_channel = dict(
+ directory=channel['directory'],
+ name=channel['name'],
+ url=channel['url'],
+ download_frequency=10,
+ )
+ request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}',
+ data=json.dumps(new_channel))
+ self.assertEqual(response.status_code, 204, response.json)
+
+ # Remove the frequency.
+ new_channel = dict(
+ directory=channel['directory'],
+ name=channel['name'],
+ url=channel['url'],
+ download_frequency=None,
+ )
+ request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}',
+ data=json.dumps(new_channel))
+ self.assertEqual(response.status_code, 204, response.json)
+
+
+def test_channel_frequency_update(download_channel, test_client, test_session):
+ """
+ A Channel's Download record is updated when the Channel's frequency is updated.
+ """
+ old_frequency = download_channel.get_download().frequency
+ assert old_frequency
+
+ data = dict(
+ directory=str(download_channel.directory.path),
+ name=download_channel.name,
+ url=download_channel.url,
+ download_frequency=100,
+ )
+ request, response = test_client.put(f'/api/videos/channels/{download_channel.link}', json=data)
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.json
+
+ download = download_channel.get_download()
+ assert download.frequency == 100
+
+ # Only one download
+ downloads = test_session.query(Download).all()
+ assert len(list(downloads)) == 1
+
+
+def test_delete_channel_delete_download(download_channel, test_session):
+ """
+ Deleting a Channel deletes it's Download.
+ """
+ downloads = test_session.query(Download).all()
+ assert all(i.url == download_channel.url for i in downloads)
+
+ delete_channel(download_channel.link)
+
+ downloads = test_session.query(Download).all()
+ assert not list(downloads)
diff --git a/modules/videos/channel/test/test_channel.py b/modules/videos/channel/test/test_channel.py
deleted file mode 100644
index 34b3d695..00000000
--- a/modules/videos/channel/test/test_channel.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import pathlib
-import unittest
-
-from modules.videos.models import Channel
-from modules.videos.test.common import create_channel_structure
-from wrolpi.db import get_db_session
-from wrolpi.test.common import wrap_test_db
-
-
-class TestChannel(unittest.TestCase):
-
- @wrap_test_db
- @create_channel_structure(
- {
- 'Foo': ['vid1.mp4'],
- }
- )
- def test_channel(self, tempdir):
- with get_db_session() as session:
- channel = session.query(Channel).one()
- self.assertIsInstance(channel.directory, pathlib.Path)
diff --git a/modules/videos/common.py b/modules/videos/common.py
index 284fca4f..a8df39b1 100644
--- a/modules/videos/common.py
+++ b/modules/videos/common.py
@@ -12,11 +12,12 @@ from PIL import Image
from sqlalchemy.orm import Session
from wrolpi import before_startup
-from wrolpi.common import sanitize_link, logger, CONFIG_PATH, get_config, iterify, chunks, get_media_directory, \
- get_absolute_media_path, minimize_dict
+from wrolpi.common import logger, CONFIG_PATH, get_config, iterify, chunks, get_media_directory, \
+ minimize_dict
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \
- ChannelLinkConflict, ChannelDirectoryConflict
+ ChannelLinkConflict, ChannelDirectoryConflict, ChannelSourceIdConflict
+from wrolpi.media_path import MediaPath
from wrolpi.vars import DOCKERIZED, DEFAULT_FILE_PERMISSIONS
from .models import Channel, Video
@@ -24,7 +25,7 @@ logger = logger.getChild(__name__)
REQUIRED_OPTIONS = ['name', 'directory']
MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count', 'link'}
-MINIMUM_INFO_JSON_KEYS = {'description', 'view_count'}
+MINIMUM_INFO_JSON_KEYS = {'description', 'view_count', 'webpage_url'}
MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'channel_id', 'favorite', 'size',
'poster_path', 'caption_path', 'video_path', 'info_json', 'channel', 'viewed', 'source_id',
'view_count'}
@@ -67,35 +68,38 @@ def import_videos_config():
config = load_channels_config()
with get_db_session(commit=True) as session:
- for section in config:
- for option in (i for i in REQUIRED_OPTIONS if i not in config[section]):
- raise ConfigError(f'Channel "{section}" is required to have "{option}"')
+ for link in config:
+ for option in (i for i in REQUIRED_OPTIONS if i not in config[link]):
+ raise ConfigError(f'Channel "{link}" is required to have "{option}"')
- name, directory = config[section]['name'], config[section]['directory']
+ name, directory = config[link]['name'], config[link]['directory']
- link = sanitize_link(name)
- matches = session.query(Channel).filter(Channel.link == link)
- if not matches.count():
- # Channel not yet in the DB, add it
+ channel = session.query(Channel).filter(Channel.link == link).one_or_none()
+ if not channel:
+ # Channel not yet in the DB, add it.
channel = Channel(link=link)
- else:
- channel = matches.one()
# Only name and directory are required
channel.name = name
channel.directory = str(directory)
- channel.calculate_duration = config[section].get('calculate_duration')
- channel.download_frequency = config[section].get('download_frequency')
- channel.generate_posters = config[section].get('generate_posters')
- channel.match_regex = config[section].get('match_regex')
- channel.skip_download_videos = list(set(config[section].get('skip_download_videos', {})))
- channel.url = config[section].get('url')
+ channel.calculate_duration = config[link].get('calculate_duration') or channel.calculate_duration
+ channel.download_frequency = config[link].get('download_frequency') or channel.download_frequency
+ channel.generate_posters = config[link].get('generate_posters') or channel.generate_posters
+ channel.match_regex = config[link].get('match_regex') or channel.match_regex
+ channel.skip_download_videos = list(set(config[link].get('skip_download_videos', {})))
+ channel.source_id = config[link].get('source_id') or channel.source_id
+ channel.url = config[link].get('url') or channel.url
+
+ if not channel.source_id and channel.url:
+ # If we can download from a channel, we must have its source_id.
+ from .downloader import get_channel_source_id
+ channel.source_id = get_channel_source_id(channel.url)
session.add(channel)
# Set favorite Videos of this Channel.
- favorites = config[section].get('favorites', {})
+ favorites = config[link].get('favorites', {})
if favorites:
videos = session.query(Video).filter(Video.source_id.in_(favorites.keys()))
for video in videos:
@@ -107,13 +111,12 @@ def import_videos_config():
VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'}
-def get_absolute_video_path(video: Video, kind: str = 'video') -> Path:
+def get_absolute_video_path(video: Video, kind: str = 'video') -> MediaPath:
if kind not in VALID_VIDEO_KINDS:
raise Exception(f'Unknown video path kind {kind}')
- directory = get_absolute_media_path(video.channel.directory)
path = getattr(video, f'{kind}_path')
- if directory and path:
- return directory / path
+ if path:
+ return path
raise UnknownFile()
@@ -123,7 +126,7 @@ get_absolute_video_description = partial(get_absolute_video_path, kind='descript
get_absolute_video_info_json = partial(get_absolute_video_path, kind='info_json')
-def get_absolute_video_files(video: Video) -> List[Path]:
+def get_absolute_video_files(video: Video) -> List[MediaPath]:
"""
Get all video files that exist.
"""
@@ -173,7 +176,7 @@ def any_extensions(filename: str, extensions: Iterable = ()):
match_video_extensions = partial(any_extensions, extensions=VIDEO_EXTENSIONS)
-def generate_video_paths(directory: Union[str, pathlib.Path], relative_to=None) -> Tuple[str, pathlib.Path]:
+def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]:
"""
Generate a list of video paths in the provided directory.
"""
@@ -182,10 +185,7 @@ def generate_video_paths(directory: Union[str, pathlib.Path], relative_to=None)
for child in directory.iterdir():
if child.is_file() and match_video_extensions(child.name):
child = child.absolute()
- if relative_to:
- yield child.relative_to(relative_to)
- else:
- yield child
+ yield child
elif child.is_dir():
yield from generate_video_paths(child)
@@ -232,7 +232,8 @@ def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]:
yield sorted(paths)[0]
-def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, link=None, directory=None):
+def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, link=None, directory=None,
+ source_id=None):
"""
Search for any channels that conflict with the provided args, raise a relevant exception if any conflicts are found.
"""
@@ -263,6 +264,10 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None,
conflicts = base_where.filter(Channel.directory == directory)
if list(conflicts):
raise ChannelDirectoryConflict()
+ if source_id:
+ conflicts = base_where.filter(Channel.source_id == source_id)
+ if list(conflicts):
+ raise ChannelSourceIdConflict()
def verify_config():
@@ -295,10 +300,14 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
"""
path = str(path)
+ ignored_directories = {
+ str(get_media_directory() / 'videos/NO CHANNEL'),
+ }
+
if os.path.isdir(path):
# The provided path is a directory, return it's subdirectories, or itself if no subdirectories exist
paths = [os.path.join(path, i) for i in os.listdir(path)]
- paths = sorted(i for i in paths if os.path.isdir(i))
+ paths = sorted(i for i in paths if os.path.isdir(i) and i not in ignored_directories)
if len(paths) == 0:
return [path]
return paths
@@ -307,7 +316,8 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
paths = os.listdir(head)
paths = [os.path.join(head, i) for i in paths]
pattern = path.lower()
- paths = sorted(i for i in paths if os.path.isdir(i) and i.lower().startswith(pattern))
+ paths = sorted(
+ i for i in paths if os.path.isdir(i) and i.lower().startswith(pattern) and i not in ignored_directories)
return paths
@@ -354,14 +364,12 @@ async def generate_bulk_posters(video_ids: List[int]):
with get_db_session(commit=True) as session:
videos = session.query(Video).filter(Video.id.in_(video_ids))
for video in videos:
- video_path = get_absolute_video_path(video)
+ video_path = video.video_path.path
poster_path = replace_extension(video_path, '.jpg')
if not poster_path.exists():
generate_video_poster(video_path)
- channel_dir = get_absolute_media_path(video.channel.directory)
- poster_path = poster_path.relative_to(channel_dir)
- video.poster_path = str(poster_path)
+ video.poster_path = poster_path
logger.info('Done generating video posters')
@@ -400,10 +408,9 @@ def bulk_validate_posters(video_ids: List[int]):
for video_ids in chunks(video_ids, 10):
for video_id in video_ids:
with get_db_session(commit=True) as session:
- video = session.query(Video).filter_by(id=video_id).one()
- channel = video.channel
+ video: Video = session.query(Video).filter_by(id=video_id).one()
- poster_path: Path = get_absolute_video_poster(video)
+ poster_path: Path = video.poster_path.path
new_poster_path = poster_path.with_suffix('.jpg')
if poster_path != new_poster_path and new_poster_path.exists():
@@ -423,10 +430,8 @@ def bulk_validate_posters(video_ids: List[int]):
else:
logger.debug(f'Poster was already valid: {new_poster_path}')
- channel_dir = get_absolute_media_path(channel.directory)
-
# Update the video with the new poster path. Mark it as validated.
- video.poster_path = str(new_poster_path.relative_to(channel_dir))
+ video.poster_path = new_poster_path
video.validated_poster = True
logger.info('Done validating video posters.')
@@ -465,6 +470,10 @@ async def get_bulk_video_info_json(video_ids: List[int]):
video = session.query(Video).filter_by(id=video_id).one()
logger.debug(f'Getting video info_json data: {video}')
+ if not video.video_path:
+ logger.warning(f'Refusing to get info_json for video without a video file: {video}')
+ continue
+
try:
info_json = video.get_info_json()
if info_json:
@@ -475,8 +484,7 @@ async def get_bulk_video_info_json(video_ids: List[int]):
video.duration = info_json.get('duration')
elif not video.duration:
# As a last resort, get duration from the video file.
- video_path = get_absolute_video_path(video)
- video.duration = get_video_duration(video_path)
+ video.duration = get_video_duration(video.video_path.path)
except Exception:
logger.warning(f'Unable to get meta data of {video}', exc_info=True)
@@ -493,16 +501,15 @@ async def get_bulk_video_size(video_ids: List[int]):
for video_id in video_ids:
video = session.query(Video).filter_by(id=video_id).one()
logger.debug(f'Getting video size: {video.id} {video.video_path}')
- video_path = get_absolute_video_path(video)
- size = video_path.stat().st_size
+ size = video.video_path.path.stat().st_size
video.size = size
logger.info('Done getting video sizes')
def update_view_count(channel_id: int):
"""
- Update view_count for all Videos in a channel.
+ Update view_count for all Videos in a channel using it's info_json file.
"""
with get_db_session() as session:
channel = session.query(Channel).filter_by(id=channel_id).one()
diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py
new file mode 100644
index 00000000..a4bdd9d0
--- /dev/null
+++ b/modules/videos/conftest.py
@@ -0,0 +1,33 @@
+import pytest
+
+from modules.videos.channel.lib import spread_channel_downloads
+from modules.videos.models import Channel
+from wrolpi.downloader import DownloadFrequency
+
+
+@pytest.fixture
+def simple_channel(test_session, test_directory) -> Channel:
+ """
+ Get a Channel with the minimum properties. This Channel has no download!
+ """
+ channel = Channel(
+ directory=test_directory,
+ name='Example Channel 1',
+ url='https://example.com/channel1',
+ download_frequency=None, # noqa
+ link='examplechannel1',
+ )
+ test_session.add(channel)
+ test_session.commit()
+ return channel
+
+
+@pytest.fixture
+def download_channel(simple_channel) -> Channel:
+ """
+ Get a test Channel that has a download frequency.
+ """
+ # Add a frequency to the test channel, then give it a download.
+ simple_channel.download_frequency = DownloadFrequency.weekly
+ spread_channel_downloads()
+ return simple_channel
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index b7e65fe9..4663ff53 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -1,21 +1,27 @@
#! /usr/bin/env python3
+import json
import pathlib
import re
-from queue import Queue
-from random import shuffle
-from typing import Tuple, List, Dict
+from abc import ABC
+from typing import Tuple, List
-from sqlalchemy import or_
from sqlalchemy.orm.exc import NoResultFound
-from youtube_dl import YoutubeDL
+from yt_dlp import YoutubeDL
+from yt_dlp.extractor import YoutubeTabIE
+from yt_dlp.utils import UnsupportedError, DownloadError
-from wrolpi.common import logger, ProgressReporter, run_after, get_absolute_media_path
-from wrolpi.dates import now, today
+from wrolpi.common import logger, sanitize_link, extract_domain, \
+ get_media_directory
+from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
-from wrolpi.errors import UnknownChannel, ChannelURLEmpty
+from wrolpi.downloader import Downloader, Download
+from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
+from wrolpi.vars import PYTEST
+from .channel.lib import create_channel, get_channel
from .common import load_downloader_config, update_view_count
-from .lib import save_channels_config, upsert_video
+from .lib import upsert_video, _refresh_videos
from .models import Video, Channel
+from .video_url_resolver import video_url_resolver
logger = logger.getChild(__name__)
ydl_logger = logger.getChild('youtube-dl')
@@ -24,17 +30,219 @@ YDL = YoutubeDL()
YDL.params['logger'] = ydl_logger
YDL.add_default_info_extractors()
+# Channels are handled differently than a single video.
+ChannelIEs = {
+ YoutubeTabIE,
+}
-def update_channel(channel=None, link: str = None):
+PREFERRED_VIDEO_EXTENSION = 'mp4'
+PREFERRED_VIDEO_FORMAT = 'best[height=720]'
+
+
+def get_no_channel_directory():
+ return get_media_directory() / 'videos/NO CHANNEL'
+
+
+class ChannelDownloader(Downloader, ABC):
+ """
+ Handling downloading of all Videos in a Channel.
+ """
+
+ @staticmethod
+ def valid_url(url) -> bool:
+ for ie in ChannelIEs:
+ if ie.suitable(url):
+ return True
+ return False
+
+ def do_download(self, download: Download):
+ """
+ Update a Channel's catalog, then schedule downloads of every missing video.
+ """
+ with get_db_session() as session:
+ channel = session.query(Channel).filter_by(url=download.url).one()
+
+ update_channel(channel)
+
+ domain = extract_domain(download.url)
+ missing_videos = find_all_missing_videos(channel.link)
+ with get_db_session(commit=True) as session:
+ for video_id, source_id, missing_video in missing_videos:
+ url = video_url_resolver(domain, missing_video)
+ # Schedule any missing videos for download.
+ self.manager.create_download(url, session, skip_download=True)
+
+ if PYTEST:
+ self.manager.do_downloads_sync()
+ else:
+ self.manager.start_downloads()
+
+ return True
+
+
+class VideoDownloader(Downloader, ABC):
+ """
+ Download a single video. Store the video in it's channel's directory, otherwise store it in `videos/NO CHANNEL`.
+ """
+
+ @staticmethod
+ def valid_url(url) -> bool:
+ """
+ Match against all Youtube-DL Info Extractors, except those that match a Channel.
+ """
+ for ie in YDL._ies.values():
+ if ie.suitable(url) and not ChannelDownloader.valid_url(url):
+ try:
+ YDL.extract_info(url, download=False, process=False)
+ return True
+ except UnsupportedError:
+ logger.debug(f'Video downloader extract_info failed for {url}')
+ return False
+ except DownloadError:
+ logger.debug(f'Video downloader extract_info failed for {url}')
+ return False
+ return False
+
+ def do_download(self, download: Download):
+ if download.attempts >= 10:
+ raise UnrecoverableDownloadError('Max download attempts reached')
+
+ url = download.url
+ try:
+ info = YDL.extract_info(url, download=False, process=False)
+ except UnsupportedError as e:
+ # Video wasn't really valid... probably the GenericIE.
+ raise UnrecoverableDownloadError() from e
+
+ channel_name = info.get('channel')
+ channel_id = info.get('channel_id')
+ channel = None
+ if channel_name or channel_id:
+ channel_url = info.get('channel_url')
+ channel = get_or_create_channel(channel_id, None, channel_url, channel_name)
+
+ # Use the default directory if this video has no channel.
+ out_dir = get_no_channel_directory()
+ if channel:
+ out_dir = channel.directory.path
+ out_dir.mkdir(exist_ok=True)
+
+ try:
+ video_path, entry = self.prepare_filename(url, out_dir)
+ # Do the real download.
+ file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
+ cmd = [
+ 'yt-dlp',
+ '-cw', # Continue downloads, do not clobber existing files.
+ '-f', PREFERRED_VIDEO_FORMAT,
+ '--write-subs',
+ '--write-auto-subs',
+ '--write-thumbnail',
+ '--write-info-json',
+ '--merge-output-format', PREFERRED_VIDEO_EXTENSION,
+ '-o', file_name_format,
+ url,
+ ]
+ return_code = self.process_runner(url, cmd, out_dir)
+
+ if return_code != 0:
+ raise ValueError(f'youtube-dl exited with {return_code}')
+
+ if not video_path.is_file():
+ raise ValueError(f'Video file could not be found! {video_path}')
+ except UnrecoverableDownloadError:
+ raise
+ except Exception as e:
+ logger.warning(f'Failed to download "{info["title"]}"', exc_info=e)
+ if _skip_download(e):
+ # The video failed to download, and the error will never be fixed. Skip it forever.
+ source_id = info.get('id')
+ logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
+ f'attempt to download it again.')
+
+ with get_db_session(commit=True) as session:
+ channel = session.query(Channel).filter_by(id=channel.id).one()
+ channel.add_video_to_skip_list(source_id)
+ return
+
+ with get_db_session(commit=True) as session:
+ # If the video is from a channel, it will already be in the database.
+ source_id = entry['id']
+ existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none()
+ id_ = existing_video.id if existing_video else None
+
+ upsert_video(session, video_path, channel, id_=id_)
+
+ # Successful download.
+ return True
+
+ @staticmethod
+ def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
+ if not out_dir.is_dir():
+ raise ValueError(f'Output directory does not exist! {out_dir=}')
+
+ # YoutubeDL expects specific options, add onto the default options
+ config = load_downloader_config()
+ options = config.copy()
+ options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}'
+ # We don't need format when getting the filename.
+ del options['format']
+
+ logger.debug(f'Downloading {url} to {out_dir}')
+
+ # Create a new YoutubeDL for the output directory.
+ ydl = YoutubeDL(options)
+ ydl.params['logger'] = ydl_logger
+ ydl.add_default_info_extractors()
+
+ # Perform the real download. This saves multiple files.
+ entry = ydl.extract_info(url, download=False, process=True)
+ final_filename = pathlib.Path(ydl.prepare_filename(entry))
+ return final_filename, entry
+
+
+channel_downloader = ChannelDownloader()
+# Videos may match the ChannelDownloader, give it a higher priority.
+video_downloader = VideoDownloader(40)
+
+
+def get_or_create_channel(source_id: str = None, link: str = None, url: str = None, name: str = None) -> Channel:
+ """
+ Attempt to find a Channel using the provided params. The params are in order of reliability.
+
+ Creates a new Channel if one cannot be found.
+ """
+ if not link and name:
+ link = sanitize_link(name)
+
+ try:
+ channel = get_channel(link=link, source_id=source_id, url=url, return_dict=False)
+ return channel
+ except UnknownChannel:
+ pass
+
+ channel_directory = get_media_directory() / f'videos/{name}'
+ data = dict(
+ name=name,
+ url=url,
+ link=link,
+ domain=extract_domain(url),
+ directory=str(channel_directory),
+ source_id=source_id,
+ )
+ channel = create_channel(data, return_dict=False)
+ # Create the directory now that the channel is approved.
+ channel_directory.mkdir(exist_ok=True)
+
+ return channel
+
+
+def update_channel(channel: Channel):
"""
Connect to the Channel's host website and pull a catalog of all videos. Insert any new videos into the DB.
It is expected that any missing videos will be downloaded later.
"""
- with get_db_session() as session:
- if not channel:
- channel = session.query(Channel).filter_by(link=link).one()
-
logger.info(f'Downloading video list for {channel.name} at {channel.url} This may take several minutes.')
info = YDL.extract_info(channel.url, download=False, process=False)
if 'url' in info:
@@ -63,16 +271,19 @@ def update_channel(channel=None, link: str = None):
logger.warning(f'entries: {entries}')
raise KeyError('No id key for entry!') from e
+ # In order to store the Video's URL, we will need a quick lookup.
+ urls = {i['id']: i.get('webpage_url') for i in entries}
+
with get_db_session(commit=True) as session:
# Get the channel in this new context.
channel = session.query(Channel).filter_by(id=channel.id).one()
channel.info_json = info
channel.info_date = now()
- channel.increment_next_download()
+ channel.source_id = info.get('id')
with get_db_curs() as curs:
- # Insert any new videos.
+ # Get all known videos in this channel.
query = 'SELECT source_id FROM video WHERE channel_id=%s AND source_id IS NOT NULL'
curs.execute(query, (channel.id,))
known_source_ids = {i[0] for i in curs.fetchall()}
@@ -82,59 +293,19 @@ def update_channel(channel=None, link: str = None):
logger.info(f'Got {len(new_source_ids)} new videos for channel {channel.name}')
channel_id = channel.id
for source_id in new_source_ids:
- session.add(Video(source_id=source_id, channel_id=channel_id))
+ url = urls.get(source_id)
+ session.add(Video(source_id=source_id, channel_id=channel_id, url=url))
+
+ # Write the Channel's info to a JSON file.
+ if channel.directory:
+ info_json_path = channel.directory.path / f'{channel.name}.info.json'
+ with info_json_path.open('wt') as fh:
+ json.dump(info, fh, indent=2)
# Update all view counts using the latest from the Channel's info_json.
update_view_count(channel_id)
-def update_channels(reporter: ProgressReporter, link: str = None):
- """Update all information for each channel. (No downloads performed)"""
-
- with get_db_session() as session:
- if session.query(Channel).count() == 0:
- raise UnknownChannel('No channels exist yet')
-
- if link:
- try:
- channel = session.query(Channel).filter_by(link=link).one()
- except NoResultFound:
- raise UnknownChannel(f'No channel with link: {link}')
- channels = [channel, ]
- else:
- channels = session.query(Channel).filter(
- Channel.url != None, # noqa
- Channel.url != '',
- or_(
- Channel.next_download == None, # noqa
- Channel.next_download <= today(),
- )
- ).all()
-
- if len(channels) == 0:
- logger.warning(f'All channels are up to date')
-
- reporter.set_progress_total(0, len(channels))
- reporter.send_progress(0, 0, f'{len(channels)} channels scheduled for update')
-
- # Randomize downloading of channels.
- shuffle(channels)
-
- logger.debug(f'Getting info for {len(channels)} channels')
- for idx, channel in enumerate(channels):
- reporter.send_progress(0, idx, f'Getting video list for {channel.name}')
- try:
- update_channel(channel)
- except Exception:
- logger.critical(f'Unable to get video list for {channel.name}', exc_info=True)
- continue
-
- if channels:
- reporter.send_progress(0, len(channels), 'Done downloading video lists')
- else:
- reporter.finish(0, 'Done downloading video lists')
-
-
def _find_all_missing_videos(link: str = None) -> List[Tuple]:
"""
Get all Video entries which don't have the required media files (i.e. hasn't been downloaded). Restrict to a
@@ -162,11 +333,12 @@ def _find_all_missing_videos(link: str = None) -> List[Tuple]:
WHERE
channel.url IS NOT NULL
AND channel.url != ''
- AND source_id IS NOT NULL
+ AND video.source_id IS NOT NULL
{where}
- AND channel_id IS NOT NULL
+ AND video.channel_id IS NOT NULL
AND (video_path IS NULL OR video_path = '' OR poster_path IS NULL OR poster_path = '')
'''
+ logger.debug(query)
curs.execute(query, params)
missing_videos = list(curs.fetchall())
return missing_videos
@@ -193,6 +365,27 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
# Get all channels while in the db context.
channels = list(channels)
+ # Check that each channel has some videos. We can't be sure what is missing if we don't know what we have.
+ for channel in channels:
+ refresh = False
+ with get_db_curs() as curs:
+ curs.execute('''
+ SELECT
+ COUNT(v.id)
+ FROM
+ channel c
+ LEFT OUTER JOIN video v on c.id = v.channel_id
+ WHERE
+ c.id = %s
+ ''', (channel.id,))
+ video_count = curs.fetchall()[0]
+ if video_count == 0:
+ # No videos for this channel. We can't be sure if any videos are missing without a refresh.
+ refresh = True
+ if refresh:
+ # Refresh this channel's videos.
+ _refresh_videos([channel.link])
+
channels = {i.id: i for i in channels}
match_regexen = {i: re.compile(j.match_regex) for i, j in channels.items() if j.match_regex}
@@ -221,31 +414,7 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
match_regex: re.compile = match_regexen.get(channel_id)
if not match_regex or (match_regex and missing_video['title'] and match_regex.match(missing_video['title'])):
# No title match regex, or the title matches the regex.
- yield channel, id_, missing_video
-
-
-def download_video(channel: Channel, video: Dict) -> Tuple[pathlib.Path, Dict]:
- """
- Download a video (and associated posters/etc) to it's channel's directory.
-
- :param channel:
- :param video: A YoutubeDL info entry dictionary
- :return:
- """
- # YoutubeDL expects specific options, add onto the default options
- config = load_downloader_config()
- options = dict(config)
- directory = get_absolute_media_path(channel.directory)
- options['outtmpl'] = f'{directory}/{config["file_name_format"]}'
-
- ydl = YoutubeDL(options)
- ydl.add_default_info_extractors()
- source_id = video['id']
- url = f'https://www.youtube.com/watch?v={source_id}'
- entry = ydl.extract_info(url, download=True, process=True)
- final_filename = ydl.prepare_filename(entry)
- final_filename = pathlib.Path(final_filename)
- return final_filename, entry
+ yield id_, source_id, missing_video
UNRECOVERABLE_ERRORS = {
@@ -268,42 +437,6 @@ def _skip_download(error):
return False
-@run_after(save_channels_config)
-def download_all_missing_videos(reporter: ProgressReporter, link: str = None):
- """Find any videos identified by the info packet that haven't yet been downloaded, download them."""
- missing_videos = list(find_all_missing_videos(link))
- reporter.set_progress_total(1, len(missing_videos))
- reporter.message(1, f'Found {len(missing_videos)} missing videos.')
-
- for idx, (channel, id_, missing_video) in enumerate(missing_videos):
- reporter.send_progress(1, idx, f'Downloading {channel.name}: {missing_video["title"]}')
- try:
- video_path, info_json = download_video(channel, missing_video)
- except Exception as e:
- logger.warning(f'Failed to download "{missing_video["title"]}"', exc_info=e)
- if _skip_download(e):
- # The video failed to download, and the error will never be fixed. Skip it forever.
- source_id = missing_video.get('id')
- logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
- f'attempt to download it again.')
-
- with get_db_session(commit=True) as session:
- channel = session.query(Channel).filter_by(id=channel.id).one()
- channel.add_video_to_skip_list(source_id)
-
- reporter.error(1, f'Failed to download "{missing_video["title"]}", see server logs...')
- continue
-
- with get_db_session(commit=True) as session:
- upsert_video(session, video_path, channel, id_=id_, info_json=info_json)
-
- reporter.finish(1, 'All videos are downloaded')
-
-
-def main(args=None):
- """Find and download any missing videos. Parse any arguments passed by the cmd-line."""
- q = Queue()
- reporter = ProgressReporter(q, 2)
- update_channels(reporter)
- download_all_missing_videos(reporter)
- return 0
+def get_channel_source_id(url: str) -> str:
+ channel_info = YDL.extract_info(url, download=False, process=False)
+ return channel_info['id']
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 64892532..d749644b 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -2,15 +2,15 @@ import asyncio
import json
import pathlib
import re
-from multiprocessing.queues import Queue
from typing import Tuple
from uuid import uuid1
from sqlalchemy.orm import Session
-from wrolpi.common import ProgressReporter, save_settings_config, get_absolute_media_path
-from wrolpi.common import logger
+from wrolpi.common import logger, chunks
+from wrolpi.common import save_settings_config
from wrolpi.db import get_db_curs, get_db_session
+from wrolpi.vars import PYTEST
from .captions import insert_bulk_captions, process_captions
from .common import generate_bulk_posters, get_bulk_video_info_json, get_bulk_video_size, \
generate_video_paths, remove_duplicate_video_paths, bulk_validate_posters, \
@@ -19,6 +19,8 @@ from .models import Channel, Video
logger = logger.getChild(__name__)
+DEFAULT_DOWNLOAD_FREQUENCY = 60 * 60 * 24 * 7 # weekly
+
def refresh_channel_video_captions() -> bool:
with get_db_curs() as curs:
@@ -86,6 +88,7 @@ def refresh_channel_info_json() -> bool:
WHERE
v.video_path IS NOT NULL
AND v.info_json_path IS NOT NULL
+ AND v.info_json_path != ''
AND (v.duration IS NULL OR v.view_count IS NULL)
'''
curs.execute(query)
@@ -128,50 +131,38 @@ def process_video_meta_data():
refresh_channel_calculate_size()
-def refresh_channel_videos(channel: Channel, reporter: ProgressReporter):
+def refresh_channel_videos(channel: Channel):
"""
Find all video files in a channel's directory. Add any videos not in the DB to the DB.
"""
- # This function is hard to predict, so we will simply progress in chunks :(
- reporter.set_progress_total(1, 7)
- reporter.send_progress(1, 0, 'Preparing channel.')
-
# Set the idempotency key so we can remove any videos not touched during this search
with get_db_curs(commit=True) as curs:
curs.execute('UPDATE video SET idempotency=NULL WHERE channel_id=%s', (channel.id,))
- reporter.send_progress(1, 1, 'Finding all videos, checking for duplicates.')
-
idempotency = str(uuid1())
- directory = get_absolute_media_path(channel.directory)
+ directory = channel.directory.path
# A set of absolute paths that exist in the file system
possible_new_paths = generate_video_paths(directory)
possible_new_paths = remove_duplicate_video_paths(possible_new_paths)
- reporter.send_progress(1, 2, 'Matching all videos to the database.')
-
# Update all videos that match the current video paths
- relative_new_paths = [str(i.relative_to(directory)) for i in possible_new_paths]
+ new_paths = [str(i) for i in possible_new_paths]
with get_db_curs(commit=True) as curs:
query = 'UPDATE video SET idempotency = %s WHERE channel_id = %s AND video_path = ANY(%s) RETURNING video_path'
- curs.execute(query, (idempotency, channel.id, relative_new_paths))
+ curs.execute(query, (idempotency, channel.id, new_paths))
existing_paths = {i for (i,) in curs.fetchall()}
- reporter.send_progress(1, 3)
-
# Get the paths for any video not yet in the DB
# (paths in DB are relative, but we need to pass an absolute path)
- new_videos = {p for p in possible_new_paths if str(p.relative_to(directory)) not in existing_paths}
+ new_videos = {i for i in possible_new_paths if str(i) not in existing_paths}
- reporter.send_progress(1, 4, f'Inserting {len(new_videos)} new videos.')
-
- for video_path in new_videos:
+ for chunk in chunks(new_videos, 20):
with get_db_session(commit=True) as session:
- upsert_video(session, pathlib.Path(video_path), channel, idempotency=idempotency)
- logger.debug(f'{channel.name}: Added {video_path}')
-
- reporter.send_progress(1, 5, 'Deleting unnecessary video entries.')
+ for video_path in chunk:
+ video_path = pathlib.Path(video_path)
+ upsert_video(session, video_path, channel, idempotency=idempotency)
+ logger.debug(f'{channel.name}: Added {video_path.name}')
with get_db_curs(commit=True) as curs:
curs.execute('DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL RETURNING id', (channel.id,))
@@ -183,14 +174,50 @@ def refresh_channel_videos(channel: Channel, reporter: ProgressReporter):
logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ')
- reporter.send_progress(1, 6, f'Updating video view counts')
-
update_view_count(channel.id)
- reporter.send_progress(1, 7, f'Processed all videos for {channel.name}')
+
+def refresh_no_channel_videos():
+ """
+ Refresh the Videos in the NO CHANNEL directory.
+ """
+ from modules.videos.downloader import get_no_channel_directory
+ directory = get_no_channel_directory()
+ if not directory.is_dir():
+ return
+
+ logger.info('Refreshing NO CHANNEL videos')
+
+ idempotency = str(uuid1())
+
+ # Get all WROLPi compatible videos, remove any duplicates (different formats).
+ possible_new_paths = generate_video_paths(directory)
+ possible_new_paths = remove_duplicate_video_paths(possible_new_paths)
+
+ new_paths = [str(i) for i in possible_new_paths]
+ with get_db_curs(commit=True) as curs:
+ query = 'UPDATE video SET idempotency = %s WHERE video_path = ANY(%s) RETURNING video_path'
+ curs.execute(query, (idempotency, new_paths))
+ existing_paths = {i for (i,) in curs.fetchall()}
+
+ new_videos = {i for i in possible_new_paths if str(i) not in existing_paths}
+
+ for chunk in chunks(new_videos, 20):
+ with get_db_session(commit=True) as session:
+ for video_path in chunk:
+ upsert_video(session, pathlib.Path(video_path), idempotency=idempotency)
+ logger.debug(f'Added NO CHANNEL video {video_path}')
+
+ with get_db_curs(commit=True) as curs:
+ curs.execute('DELETE FROM video WHERE channel_id IS NULL AND idempotency IS NULL RETURNING id')
+ deleted_count = len(curs.fetchall())
+
+ if deleted_count:
+ deleted_status = f'Deleted {deleted_count} video records in NO CHANNEL.'
+ logger.info(deleted_status)
-def _refresh_videos(q: Queue, channel_links: list = None):
+def _refresh_videos(channel_links: list = None):
"""
Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the
file system.
@@ -201,10 +228,6 @@ def _refresh_videos(q: Queue, channel_links: list = None):
"""
logger.info('Refreshing video files')
with get_db_session() as session:
- reporter = ProgressReporter(q, 2)
- reporter.code('refresh-started')
- reporter.set_progress_total(0, session.query(Channel).count())
-
if channel_links:
channels = session.query(Channel).filter(Channel.link.in_(channel_links))
else:
@@ -218,14 +241,15 @@ def _refresh_videos(q: Queue, channel_links: list = None):
raise Exception(f'No channels in DB. Have you created any?')
for idx, channel in enumerate(channels):
- reporter.send_progress(0, idx, f'Checking {channel.name} directory for new videos')
- refresh_channel_videos(channel, reporter)
+ refresh_channel_videos(channel)
+
+ if not channel_links:
+ # Refresh NO CHANNEL videos when not refreshing a specific channel.
+ refresh_no_channel_videos()
# Fill in any missing data for all videos.
- process_video_meta_data()
-
- reporter.send_progress(0, len(channels), 'All videos refreshed.')
- reporter.code('refresh-complete')
+ if not PYTEST:
+ process_video_meta_data()
def get_channels_config(session: Session) -> dict:
@@ -319,43 +343,20 @@ async def get_statistics():
return ret
-async def get_download_history():
- """
- Get the info_date for all channels that have been downloaded. Order by the most recent download first.
- """
- query = '''
- SELECT
- link, name, info_date
- FROM
- channel
- WHERE
- info_date IS NOT NULL
- ORDER BY channel.info_date DESC
- '''
- with get_db_curs() as curs:
- curs.execute(query)
- results = [dict(i) for i in curs.fetchall()]
- return results
-
-
NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{11})_)?(.*)\.'
r'(jpg|webp|flv|mp4|part|info\.json|description|webm|..\.srt|..\.vtt)')
-def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel, idempotency: str = None,
- skip_captions=False,
- id_: str = None, info_json: dict = None) -> Video:
+def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = None, idempotency: str = None,
+ skip_captions=False, id_: str = None) -> Video:
"""
Insert a video into the DB. Also, find any meta-files near the video file and store them on the video row.
If id_ is provided, update that entry.
"""
- channel_dir = get_absolute_media_path(channel.directory)
- poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path, relative_to=channel_dir)
-
- # Video paths should be relative to the channel's directory
- if video_path.is_absolute():
- video_path = video_path.relative_to(channel_dir)
+ if not video_path.is_absolute():
+ raise ValueError(f'Video path is not absolute: {video_path}')
+ poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path)
name_match = NAME_PARSER.match(video_path.name)
_ = upload_date = source_id = title = ext = None
@@ -369,28 +370,33 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel, i
upload_date = None
duration = None
+ url = None
if info_json_path:
- path = (channel_dir / info_json_path).absolute()
try:
- with open(path) as fh:
+ with info_json_path.open('rt') as fh:
json_contents = json.load(fh)
- duration = json_contents['duration']
+ duration = json_contents.get('duration')
+ url = json_contents.get('webpage_url')
except json.decoder.JSONDecodeError:
- logger.warning(f'Failed to load JSON file to get duration: {path}')
+ logger.warning(f'Failed to load JSON file to get duration: {info_json_path}')
+
+ size = video_path.stat().st_size
video_dict = dict(
- channel_id=channel.id,
+ caption_path=str(caption_path) if caption_path else None,
+ channel_id=channel.id if channel else None,
description_path=str(description_path) if description_path else None,
+ duration=duration,
ext=ext,
+ idempotency=idempotency,
+ info_json_path=str(info_json_path) if info_json_path else None,
poster_path=str(poster_path) if poster_path else None,
+ size=size,
source_id=source_id,
title=title,
upload_date=upload_date,
+ url=url,
video_path=str(video_path),
- caption_path=str(caption_path) if caption_path else None,
- idempotency=idempotency,
- info_json_path=str(info_json_path) if info_json_path else None,
- duration=duration,
)
if id_:
@@ -410,8 +416,7 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel, i
return video
-def find_meta_files(path: pathlib.Path, relative_to=None) -> Tuple[
- pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]:
+def find_meta_files(path: pathlib.Path) -> Tuple[pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]:
"""
Find all files that share a file's full path, except their extensions. It is assumed that file with the
same name, but different extension is related to that file. A None will be yielded if the meta file doesn't exist.
@@ -429,11 +434,7 @@ def find_meta_files(path: pathlib.Path, relative_to=None) -> Tuple[
for meta_ext in meta_exts:
meta_path = path.with_suffix(meta_ext)
if meta_path.exists():
- if relative_to:
- yield meta_path.relative_to(relative_to)
- break
- else:
- yield meta_path
- break
+ yield meta_path
+ break
else:
yield None
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 5c9ac131..f71925f3 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -1,27 +1,20 @@
import json
-from datetime import timedelta, datetime
+from datetime import datetime
from pathlib import Path
-from typing import Optional
+from typing import Optional, List
-from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, Computed, types
+from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, Computed
from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm.collections import InstrumentedList
-from wrolpi.common import Base, tsvector, ModelHelper, PathColumn, get_absolute_media_path
-from wrolpi.dates import now, today, TZDateTime
+from wrolpi.common import Base, tsvector, ModelHelper, logger
+from wrolpi.dates import now, TZDateTime
from wrolpi.db import get_db_curs
-from wrolpi.errors import UnknownVideo, NoFrequency, UnknownFile, UnknownDirectory
+from wrolpi.downloader import Download
+from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory
+from wrolpi.media_path import MediaPathType, MediaPath
-
-class ChannelPath(types.TypeDecorator):
- impl = types.String
-
- def process_bind_param(self, value, dialect):
- return str(value) if value else None
-
- def process_result_value(self, value, dialect):
- if value:
- return Path(value)
+logger = logger.getChild(__name__)
class Video(ModelHelper, Base):
@@ -32,12 +25,12 @@ class Video(ModelHelper, Base):
idempotency = Column(String)
# File paths
- caption_path = Column(ChannelPath)
- description_path = Column(ChannelPath)
+ caption_path = Column(MediaPathType)
+ description_path = Column(MediaPathType)
ext = Column(String)
- info_json_path = Column(ChannelPath)
- poster_path = Column(ChannelPath)
- video_path = Column(ChannelPath)
+ info_json_path = Column(MediaPathType)
+ poster_path = Column(MediaPathType)
+ video_path = Column(MediaPathType)
caption = Column(String)
duration = Column(Integer)
@@ -49,12 +42,13 @@ class Video(ModelHelper, Base):
validated_poster = Column(Boolean, default=False)
viewed = Column(TZDateTime)
view_count = Column(Integer)
+ url = Column(String)
textsearch = Column(tsvector, Computed('''to_tsvector('english'::regconfig,
((COALESCE(title, ''::text) || ' '::text) ||
COALESCE(caption, ''::text)))'''))
def __repr__(self):
- return f'