mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
Creating new /api/archive/upload which accepts a file from the SingleFile browser extension and converts it to an Archive.
This commit is contained in:
parent
01338e4d89
commit
531ec3d65e
62 changed files with 1000 additions and 562 deletions
|
|
@ -1,18 +1,15 @@
|
|||
import asyncio
|
||||
import json
|
||||
import pathlib
|
||||
import tempfile
|
||||
from abc import ABC
|
||||
from json import JSONDecodeError
|
||||
from typing import List, Tuple, Iterable
|
||||
|
||||
from selenium import webdriver
|
||||
from sqlalchemy import not_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
|
||||
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM
|
||||
from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
|
||||
slow_logger, html_screenshot, get_title_from_html
|
||||
slow_logger, get_title_from_html
|
||||
from wrolpi.db import optional_session, get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
|
|
@ -38,6 +35,11 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
def __repr__(self):
|
||||
return f'<ArchiveDownloader>'
|
||||
|
||||
@optional_session
|
||||
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
|
||||
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
|
||||
return file_groups
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
if download.attempts > 3:
|
||||
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
|
||||
|
|
@ -45,12 +47,13 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
if DOCKERIZED or PYTEST:
|
||||
# Perform the archive in the Archive docker container. (Typically in the development environment).
|
||||
singlefile, readability, screenshot = await request_archive(download.url)
|
||||
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
|
||||
archive_id = archive.id
|
||||
else:
|
||||
# Perform the archive using locally installed executables.
|
||||
singlefile, readability, screenshot = await self.do_archive(download)
|
||||
|
||||
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
|
||||
archive_id = archive.id
|
||||
singlefile = await self.do_singlefile(download)
|
||||
archive = await lib.singlefile_to_archive(singlefile)
|
||||
archive_id = archive.id
|
||||
|
||||
with get_db_session() as session:
|
||||
archive = Archive.find_by_id(archive_id, session)
|
||||
|
|
@ -67,12 +70,7 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
|
||||
return DownloadResult(success=True, location=f'/archive/{archive.id}')
|
||||
|
||||
@optional_session
|
||||
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
|
||||
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
|
||||
return file_groups
|
||||
|
||||
async def do_singlefile(self, download: Download) -> Tuple[bytes, dict]:
|
||||
async def do_singlefile(self, download: Download) -> bytes:
|
||||
"""Create a Singlefile from the archive's URL."""
|
||||
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
|
||||
str(SINGLE_FILE_BIN),
|
||||
|
|
@ -83,115 +81,26 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
'--load-deferred-images-dispatch-scroll-event',
|
||||
download.url,
|
||||
)
|
||||
return_code, logs, stdout = await self.process_runner(
|
||||
download.id,
|
||||
download.url,
|
||||
cmd,
|
||||
pathlib.Path('/home/wrolpi'),
|
||||
)
|
||||
cwd = pathlib.Path('/home/wrolpi')
|
||||
cwd = cwd if cwd.is_dir() else None
|
||||
result = await self.process_runner(download, cmd, cwd)
|
||||
|
||||
stderr = logs['stderr'].decode()
|
||||
log_output = stderr or stdout.decode() or 'No stderr or stdout!'
|
||||
stderr = result.stderr.decode()
|
||||
log_output = stderr or result.stdout.decode() or 'No stderr or stdout!'
|
||||
|
||||
if return_code != 0:
|
||||
if result.return_code != 0:
|
||||
e = ChildProcessError(log_output[:1000])
|
||||
raise RuntimeError(f'singlefile exited with {return_code}') from e
|
||||
raise RuntimeError(f'singlefile exited with {result.return_code}') from e
|
||||
|
||||
if not stdout:
|
||||
if not result.stdout:
|
||||
e = ChildProcessError(log_output[:1000])
|
||||
raise RuntimeError(f'Singlefile created was empty: {download.url}') from e
|
||||
|
||||
if SINGLEFILE_HEADER.encode() not in stdout:
|
||||
if SINGLEFILE_HEADER.encode() not in result.stdout:
|
||||
e = ChildProcessError(log_output[:1000])
|
||||
raise RuntimeError(f'Singlefile created was invalid: {download.url}') from e
|
||||
|
||||
return stdout, logs
|
||||
|
||||
async def do_readability(self, download: Download, html: bytes) -> dict:
|
||||
"""Extract the readability dict from the provided HTML."""
|
||||
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
|
||||
fh.write(html)
|
||||
|
||||
cmd = (READABILITY_BIN, fh.name, download.url)
|
||||
logger.debug(f'readability cmd: {cmd}')
|
||||
return_code, logs, stdout = await self.process_runner(
|
||||
download.id,
|
||||
download.url,
|
||||
cmd,
|
||||
pathlib.Path('/home/wrolpi'),
|
||||
)
|
||||
if return_code == 0:
|
||||
if not stdout:
|
||||
raise RuntimeError('readability stdout was empty')
|
||||
try:
|
||||
readability = json.loads(stdout)
|
||||
except TypeError or JSONDecodeError:
|
||||
# JSON was invalid.
|
||||
e = ChildProcessError(stdout.decode() if stdout else 'No stdout')
|
||||
if logs and (stderr := logs.get('stderr')):
|
||||
e = ChildProcessError(stderr.decode())
|
||||
raise RuntimeError(f'Failed to extract readability from {download.url}') from e
|
||||
logger.debug(f'done readability for {download.url}')
|
||||
return readability
|
||||
else:
|
||||
logger.error(f'Failed to extract readability for {download.url}')
|
||||
e = ChildProcessError(stdout.decode() if stdout else 'No stdout')
|
||||
if logs and (stderr := logs.get('stderr')):
|
||||
e = ChildProcessError(stderr.decode())
|
||||
raise RuntimeError(f'Failed to extract readability for {download.url} got {return_code}') from e
|
||||
|
||||
@staticmethod
|
||||
async def do_screenshot_url(download: Download) -> bytes:
|
||||
"""Use Chromium to get a screenshot of the Download's URL."""
|
||||
# Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of
|
||||
# the page.
|
||||
options = webdriver.ChromeOptions()
|
||||
options.add_argument('headless')
|
||||
options.add_argument('disable-gpu')
|
||||
options.add_argument('window-size=1280x720')
|
||||
|
||||
driver = webdriver.Chrome(chrome_options=options)
|
||||
driver.get(download.url)
|
||||
screenshot = driver.get_screenshot_as_png()
|
||||
return screenshot
|
||||
|
||||
async def do_archive(self, download: Download) -> Tuple[bytes, dict, bytes]:
|
||||
"""Use locally installed executables to create an Archive.
|
||||
|
||||
Creates a Singlefile, readability file, and screenshot file.
|
||||
|
||||
@warning: Will not raise errors if readability or screenshot cannot be extracted.
|
||||
"""
|
||||
singlefile, logs = await self.do_singlefile(download)
|
||||
|
||||
# Extract Readability from the Singlefile.
|
||||
try:
|
||||
readability = await self.do_readability(download, singlefile)
|
||||
except RuntimeError:
|
||||
# Readability is not required.
|
||||
readability = None
|
||||
|
||||
screenshot = b''
|
||||
try:
|
||||
# Screenshot the Singlefile first.
|
||||
screenshot = html_screenshot(singlefile)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to screenshot file for {download.url}', exc_info=e)
|
||||
|
||||
if not screenshot:
|
||||
# Screenshot of singlefile failed. Download the URL again.
|
||||
try:
|
||||
screenshot = await self.do_screenshot_url(download)
|
||||
except Exception as e:
|
||||
# Screenshot failed.
|
||||
logger.error(f'Failed to screenshot {download.url}', exc_info=e)
|
||||
|
||||
singlefile_len = len(singlefile) if singlefile else None
|
||||
readability_len = len(readability) if readability else None
|
||||
screenshot_len = len(screenshot) if screenshot else None
|
||||
logger.debug(f'do_archive of {download.url} finished: {singlefile_len=} {readability_len=} {screenshot_len=}')
|
||||
|
||||
return singlefile, readability, screenshot
|
||||
return result.stdout
|
||||
|
||||
|
||||
archive_downloader = ArchiveDownloader()
|
||||
|
|
@ -258,7 +167,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
|||
archive.validate()
|
||||
archive.flush()
|
||||
|
||||
file_group.title = file_group.a_text = title
|
||||
file_group.title = file_group.a_text = title or archive.file_group.title
|
||||
file_group.d_text = contents
|
||||
file_group.data = {
|
||||
'id': archive.id,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@ from sanic_ext.extensions.openapi import openapi
|
|||
from wrolpi.api_utils import json_response
|
||||
from wrolpi.common import logger, wrol_mode_check, api_param_limiter
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.schema import JSONErrorResponse
|
||||
from wrolpi.switches import register_switch_handler
|
||||
from . import lib, schema
|
||||
|
||||
NAME = 'archive'
|
||||
|
|
@ -29,7 +31,7 @@ async def get_archive(_: Request, archive_id: int):
|
|||
|
||||
@archive_bp.delete('/<archive_ids:int>', name='archive_delete_one')
|
||||
@archive_bp.delete('/<archive_ids:[0-9,]+>', name='archive_delete_many')
|
||||
@openapi.description('Delete an individual archive')
|
||||
@openapi.description('Delete Archives')
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
@wrol_mode_check
|
||||
async def delete_archive(_: Request, archive_ids: str):
|
||||
|
|
@ -70,3 +72,38 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
|
|||
body.headline)
|
||||
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
|
||||
return json_response(ret)
|
||||
|
||||
|
||||
@register_switch_handler('singlefile_upload_switch_handler')
|
||||
async def singlefile_upload_switch_handler(singlefile):
|
||||
archive = await lib.singlefile_to_archive(singlefile)
|
||||
logger.info(f'Created Archive from upload: {archive}')
|
||||
name = archive.file_group.title or archive.file_group.url
|
||||
Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location)
|
||||
|
||||
|
||||
@archive_bp.get('/upload')
|
||||
@openapi.definition(
|
||||
summary='A message to confirm to the user that they have the correct upload URL.'
|
||||
)
|
||||
async def get_upload_singlefile(request: Request):
|
||||
return response.text('This is the URL to upload files using the SingleFile browser extension.'
|
||||
' This requires a POST request.')
|
||||
|
||||
|
||||
@archive_bp.post('/upload')
|
||||
@openapi.definition(
|
||||
summary='Upload SingleFile from SingleFile browser extension and convert it to an Archive.'
|
||||
)
|
||||
async def post_upload_singlefile(request: Request):
|
||||
url = request.form['url'][0]
|
||||
singlefile = request.files['singlefile_contents'][0].body
|
||||
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url}')
|
||||
|
||||
# Get the URL to ensure that the singlefile is valid.
|
||||
lib.get_url_from_singlefile(singlefile)
|
||||
|
||||
# Send processing to background task so the extension can continue.
|
||||
singlefile_upload_switch_handler.activate_switch(dict(singlefile=singlefile))
|
||||
# Return empty json response because SingleFile extension expects a JSON response.
|
||||
return json_response(dict(), status=HTTPStatus.OK)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,15 @@
|
|||
import base64
|
||||
import base64
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shlex
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from json import JSONDecodeError
|
||||
from typing import Optional, Tuple, List, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
|
@ -14,13 +19,14 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from modules.archive.models import Domain, Archive
|
||||
from wrolpi import dates
|
||||
from wrolpi.cmd import READABILITY_BIN, run_command
|
||||
from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \
|
||||
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config
|
||||
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config, html_screenshot
|
||||
from wrolpi.dates import now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
|
||||
from wrolpi.tags import tag_append_sub_select_where
|
||||
from wrolpi.vars import PYTEST
|
||||
from wrolpi.vars import PYTEST, DOCKERIZED
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -106,11 +112,11 @@ def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles:
|
|||
ARCHIVE_TIMEOUT = Seconds.minute * 10 # Wait at most 10 minutes for response.
|
||||
|
||||
|
||||
async def request_archive(url: str) -> Tuple[str, Optional[str], Optional[str]]:
|
||||
async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Optional[dict], Optional[str]]:
|
||||
"""Send a request to the archive service to archive the URL."""
|
||||
logger.info(f'Sending archive request to archive service: {url}')
|
||||
|
||||
data = {'url': url}
|
||||
data = dict(url=url, singlefile=singlefile)
|
||||
try:
|
||||
async with aiohttp_post(f'{ARCHIVE_SERVICE}/json', json_=data, timeout=ARCHIVE_TIMEOUT) as response:
|
||||
status = response.status
|
||||
|
|
@ -229,9 +235,9 @@ def get_or_create_domain(session: Session, url) -> Domain:
|
|||
SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n')
|
||||
|
||||
|
||||
def get_url_from_singlefile(html: bytes) -> str:
|
||||
def get_url_from_singlefile(html: bytes | str) -> str:
|
||||
"""Extract URL from SingleFile contents."""
|
||||
html = html.decode()
|
||||
html = html.decode() if isinstance(html, bytes) else html
|
||||
if SINGLEFILE_HEADER not in html:
|
||||
raise RuntimeError(f'Not a singlefile!')
|
||||
|
||||
|
|
@ -313,7 +319,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
|
|||
except json.decoder.JSONDecodeError:
|
||||
# Was not valid JSON.
|
||||
schema = None
|
||||
if isinstance(schema, dict) and schema.get('@context') in ('https://schema.org', 'http://schema.org'):
|
||||
if isinstance(schema, dict) and '://schema.org' in schema.get('@context'):
|
||||
# Found https://schema.org/
|
||||
if headline := schema.get('headline'):
|
||||
metadata.title = metadata.title or headline
|
||||
|
|
@ -347,6 +353,17 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
|
|||
metadata.author = author
|
||||
else:
|
||||
logger.warning(f'Unable to parse author schema: {author}')
|
||||
elif authors := schema.get('authors'):
|
||||
# Use the first Author.
|
||||
author = authors[0]
|
||||
|
||||
if isinstance(author, dict):
|
||||
author = author.get('name') or author
|
||||
|
||||
if isinstance(author, str):
|
||||
metadata.author = author
|
||||
else:
|
||||
logger.warning(f'Unable to parse author schema: {author}')
|
||||
|
||||
# Assume UTC if no timezone.
|
||||
if metadata.published_datetime and not metadata.published_datetime.tzinfo and assume_utc:
|
||||
|
|
@ -368,7 +385,7 @@ def get_archive(session, archive_id: int) -> Archive:
|
|||
|
||||
|
||||
def delete_archives(*archive_ids: List[int]):
|
||||
"""Delete an Archive and all of it's files."""
|
||||
"""Delete an Archive and all of its files."""
|
||||
with get_db_session(commit=True) as session:
|
||||
archives: List[Archive] = list(session.query(Archive).filter(Archive.id.in_(archive_ids)))
|
||||
if not archives:
|
||||
|
|
@ -535,3 +552,75 @@ async def search_domains_by_name(name: str, limit: int = 5, session: Session = N
|
|||
.limit(limit) \
|
||||
.all()
|
||||
return domains
|
||||
|
||||
|
||||
async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
|
||||
"""Extract the readability dict from the provided HTML."""
|
||||
with tempfile.NamedTemporaryFile('wb', suffix='.html') as singlefile_file:
|
||||
singlefile_file.write(html.encode() if isinstance(html, str) else html)
|
||||
singlefile_file.flush()
|
||||
os.fsync(singlefile_file.fileno())
|
||||
|
||||
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
|
||||
READABILITY_BIN, singlefile_file.name, shlex.quote(url))
|
||||
# Docker containers may not have this directory. But, this directory is necessary on RPi.
|
||||
cwd = '/home/wrolpi' if os.path.isdir('/home/wrolpi') else None
|
||||
result = await run_command(cmd, cwd=cwd, timeout=timeout, debug=True)
|
||||
|
||||
logger.debug(f'readability for {url} exited with {result.return_code}')
|
||||
stdout, stderr = result.stdout.decode(), result.stderr.decode()
|
||||
if result.return_code == 0:
|
||||
if not stdout:
|
||||
raise RuntimeError('readability stdout was empty')
|
||||
try:
|
||||
readability = json.loads(stdout)
|
||||
except TypeError or JSONDecodeError:
|
||||
# JSON was invalid.
|
||||
e = ChildProcessError(stdout if stdout else 'No stdout')
|
||||
if stderr:
|
||||
e = ChildProcessError(stderr)
|
||||
raise RuntimeError(f'Failed to extract readability from {url}') from e
|
||||
logger.debug(f'done readability for {url}')
|
||||
return readability
|
||||
else:
|
||||
logger.error(f'Failed to extract readability for {url}')
|
||||
e = ChildProcessError(stdout if stdout else 'No stdout')
|
||||
if stderr:
|
||||
e = ChildProcessError(stderr)
|
||||
raise RuntimeError(f'Failed to extract readability for {url} got {result.return_code}') from e
|
||||
|
||||
|
||||
async def singlefile_to_archive(singlefile: bytes) -> Archive:
|
||||
"""
|
||||
Convert a SingleFile to an Archive.
|
||||
|
||||
This is done by extracting readability, creating a screenshot, then attaching them to an Archive/FileGroup.
|
||||
"""
|
||||
# Get URL first because it does some simple checking of `singlefile`
|
||||
url = get_url_from_singlefile(singlefile)
|
||||
singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile
|
||||
|
||||
if DOCKERIZED:
|
||||
# Perform the archive in the Archive docker container. (Typically in the development environment).
|
||||
singlefile = base64.b64encode(singlefile).decode()
|
||||
logger.debug(f'singlefile_to_archive sending to archive service: {url}')
|
||||
singlefile, readability, screenshot = await request_archive(url, singlefile=singlefile)
|
||||
else:
|
||||
# JSON from readability-extractor
|
||||
readability = dict()
|
||||
try:
|
||||
logger.debug(f'singlefile_to_archive extracting readability: {url}')
|
||||
readability = await html_to_readability(singlefile, url)
|
||||
except RuntimeError as e:
|
||||
logger.error(f'Failed to extract readability from: {url}', exc_info=e)
|
||||
|
||||
screenshot = None
|
||||
try:
|
||||
logger.debug(f'singlefile_to_archive creating screenshot: {url}')
|
||||
screenshot = html_screenshot(singlefile)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract screenshot from: {url}', exc_info=e)
|
||||
|
||||
logger.trace(f'singlefile_to_archive modeling: {url}')
|
||||
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot)
|
||||
return archive
|
||||
|
|
|
|||
|
|
@ -318,6 +318,11 @@ class Archive(Base, ModelHelper):
|
|||
def add_tag(self, tag_id_or_name: int | str, session: Session = None) -> TagFile:
|
||||
return self.file_group.add_tag(tag_id_or_name, session)
|
||||
|
||||
@property
|
||||
def location(self):
|
||||
"""The location where this Archive can be viewed in the UI."""
|
||||
return f'/archive/{self.id}'
|
||||
|
||||
|
||||
class Domain(Base, ModelHelper):
|
||||
__tablename__ = 'domains' # plural to avoid conflict
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ from http import HTTPStatus
|
|||
|
||||
import pytest
|
||||
|
||||
from modules.archive import lib
|
||||
from modules.archive import lib, Archive
|
||||
from wrolpi.test.common import skip_circleci
|
||||
|
||||
|
||||
def check_results(test_client, data, ids):
|
||||
|
|
@ -227,3 +228,31 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
|
|||
request, response = test_client.get(f'/api/archive/domains')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['domains'] == []
|
||||
|
||||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form,
|
||||
await_switches, events_history):
|
||||
"""Test converting a SingleFile from the SingleFile browser extension to an Archive."""
|
||||
singlefile_contents = singlefile_contents_factory(title='upload title', url='https://example.com/single-file-url')
|
||||
forms = [
|
||||
dict(name='url', value='https://example.com/form-url'),
|
||||
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents')
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
await await_switches()
|
||||
|
||||
archive, = test_session.query(Archive).all()
|
||||
assert archive.singlefile_path.is_file(), 'Singlefile should have been saved.'
|
||||
assert archive.readability_path.is_file(), 'Readability should be extracted.'
|
||||
assert archive.readability_json_path.is_file(), 'Readability json should be extracted.'
|
||||
assert archive.screenshot_path.is_file(), 'Screenshot should have been created'
|
||||
assert archive.file_group.title == 'upload title', 'Title was not correct'
|
||||
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
|
||||
events = list(events_history)
|
||||
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'
|
||||
|
|
|
|||
|
|
@ -561,6 +561,35 @@ async def test_refresh_archives_index(test_session, make_files_structure):
|
|||
assert archive.file_group.indexed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_meta(async_client, test_session, make_files_structure):
|
||||
# The start of a typical singlefile html file.
|
||||
singlefile_contents = '''<!DOCTYPE html> <html lang="en">
|
||||
<script data-vue-meta="ssr" type="application/ld+json">
|
||||
{"@context":"https://schema.org/","@type":"NewsArticle","headline":"The Headline","description":"The Description","authors":[{"name":"A.B.C.","@type":"Person"}],"datePublished":"2021-06-20T10:00"}
|
||||
</script>
|
||||
</html>'''
|
||||
|
||||
singlefile, *_ = make_files_structure({
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.png': None,
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents',
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
|
||||
})
|
||||
|
||||
await files_lib.refresh_files()
|
||||
|
||||
archive: Archive = test_session.query(Archive).one()
|
||||
assert archive.singlefile_path == singlefile
|
||||
|
||||
assert archive.file_group.author == 'A.B.C.'
|
||||
assert archive.file_group.published_datetime == datetime(2021, 6, 20, 10, 0, tzinfo=pytz.utc)
|
||||
|
||||
assert archive.file_group.title == 'The Headline'
|
||||
assert archive.file_group.b_text == 'The Description'
|
||||
assert archive.file_group.indexed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_archives_deleted_singlefile(test_session, make_files_structure, singlefile_contents_factory):
|
||||
"""Removing a Singlefile file from a FileGroup makes that group no longer an Archive."""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import wrolpi
|
||||
from wrolpi.db import get_db_session
|
||||
from .api import inventory_bp
|
||||
from .inventory import logger, DEFAULT_CATEGORIES, DEFAULT_INVENTORIES
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import asyncio
|
||||
import subprocess
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
|
|
@ -8,9 +8,9 @@ from sqlalchemy.orm import Session
|
|||
from modules.map.models import MapFile
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.cmd import SUDO_BIN
|
||||
from wrolpi.cmd import SUDO_BIN, run_command
|
||||
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config
|
||||
from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp
|
||||
from wrolpi.dates import timedelta_to_timestamp, seconds_to_timestamp
|
||||
from wrolpi.db import optional_session, get_db_session
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.vars import PROJECT_DIR, IS_RPI5
|
||||
|
|
@ -37,7 +37,7 @@ def is_pbf_file(pbf: Path) -> bool:
|
|||
|
||||
|
||||
def is_dump_file(path: Path) -> bool:
|
||||
"""Uses file command to check type of a file. Returns True if a file is a Postgresql dump file."""
|
||||
"""Uses file command to check the type of file. Returns True if a file is a Postgresql dump file."""
|
||||
cmd = ('/usr/bin/file', path)
|
||||
try:
|
||||
output = subprocess.check_output(cmd)
|
||||
|
|
@ -188,24 +188,22 @@ async def run_import_command(*paths: Path) -> int:
|
|||
|
||||
paths = ' '.join(str(i) for i in paths)
|
||||
# Run with sudo so renderd can be restarted.
|
||||
cmd = f'{SUDO_BIN} {PROJECT_DIR}/scripts/import_map.sh {paths}'
|
||||
cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', paths)
|
||||
import_logger.warning(f'Running map import command: {cmd}')
|
||||
start = now()
|
||||
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
|
||||
result = await run_command(cmd, debug=True)
|
||||
|
||||
stdout, stderr = await proc.communicate()
|
||||
elapsed = now() - start
|
||||
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
|
||||
import_logger.info(f'{success} import of {repr(paths)} took {timedelta_to_timestamp(elapsed)}')
|
||||
if proc.returncode != 0:
|
||||
success = 'Successful' if result.return_code == 0 else 'Unsuccessful'
|
||||
import_logger.info(
|
||||
f'{success} import of {repr(paths)} took {timedelta_to_timestamp(timedelta(seconds=result.elapsed))}')
|
||||
if result.return_code != 0:
|
||||
# Log all lines. Truncate long lines.
|
||||
for line in stdout.decode().splitlines():
|
||||
for line in result.stdout.decode().splitlines():
|
||||
import_logger.debug(line[:500])
|
||||
for line in stderr.decode().splitlines():
|
||||
for line in result.stderr.decode().splitlines():
|
||||
import_logger.error(line[:500])
|
||||
raise ValueError(f'Importing map file failed with return code {proc.returncode}')
|
||||
raise ValueError(f'Importing map file failed with return code {result.return_code}')
|
||||
|
||||
return int(elapsed.total_seconds())
|
||||
return result.elapsed
|
||||
|
||||
|
||||
@optional_session
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ def assert_map_file_status(test_client, expected, status_code=HTTPStatus.OK):
|
|||
|
||||
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
|
||||
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
|
||||
def test_status_and_import(test_client, test_session, make_files_structure, mock_create_subprocess_shell):
|
||||
def test_status_and_import(test_client, test_session, make_files_structure, mock_run_command):
|
||||
"""PBF files can be imported, and the status of the import can be monitored."""
|
||||
pbf1, pbf2, dump1 = make_files_structure([
|
||||
'map/pbf/some-country.osm.pbf',
|
||||
|
|
@ -32,11 +32,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(pbf1), ], }
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(pbf1), ], }
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
expected = [
|
||||
{'imported': False, 'path': 'map/dump/dumpy.dump'},
|
||||
|
|
@ -45,11 +43,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(pbf1), str(pbf2)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(pbf1), str(pbf2)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
expected = [
|
||||
{'imported': False, 'path': 'map/dump/dumpy.dump'},
|
||||
|
|
@ -58,11 +54,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(dump1), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(dump1), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
expected = [
|
||||
{'imported': True, 'path': 'map/dump/dumpy.dump'},
|
||||
|
|
@ -74,7 +68,7 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
|
|||
|
||||
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
|
||||
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
|
||||
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_create_subprocess_shell):
|
||||
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_run_command):
|
||||
"""Multiple PBFs can be imported. Importing a second time overwrites the previous imports."""
|
||||
pbf1, pbf2, pbf3, dump = make_files_structure([
|
||||
'map/pbf/country1.osm.pbf',
|
||||
|
|
@ -83,11 +77,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
|
|||
'map/dumpy.dump',
|
||||
])
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(pbf2), str(pbf1)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(pbf2), str(pbf1)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Two PBFs can be merged and imported.
|
||||
expected = [
|
||||
|
|
@ -98,11 +90,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(pbf3), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(pbf3), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# New imports overwrite old imports.
|
||||
expected = [
|
||||
|
|
@ -113,12 +103,10 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
# Can't mix PBF and dump.
|
||||
body = {'files': [str(dump), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
# Can't mix PBF and dump.
|
||||
body = {'files': [str(dump), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Dump does not overwrite.
|
||||
expected = [
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ def test_get_map_paths(test_directory, make_files_structure):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_import_command(test_directory, mock_create_subprocess_shell):
|
||||
async def test_run_import_command(test_directory, mock_run_command, run_command_bad_result):
|
||||
"""
|
||||
Import map files. Files to import are checked for validity before importing. Any errors returned by
|
||||
asyncio.create_subprocess_shell are caught and reported.
|
||||
|
|
@ -38,22 +38,14 @@ async def test_run_import_command(test_directory, mock_create_subprocess_shell):
|
|||
dump_file = test_directory / 'dumpy.dump'
|
||||
dump_file.touch()
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
|
||||
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
|
||||
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
|
||||
# Run the import, it succeeds.
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
|
||||
communicate_return=(b'out', b'error')
|
||||
)
|
||||
mock_is_pbf_file.return_value = True
|
||||
await lib.run_import_command(pbf_file)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
|
||||
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
|
||||
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
|
||||
# Run the import, it fails.
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
|
||||
communicate_return=(b'out', b'error'),
|
||||
return_code=1,
|
||||
)
|
||||
mock_run_command(run_command_bad_result)
|
||||
mock_is_pbf_file.return_value = True
|
||||
with pytest.raises(ValueError):
|
||||
await lib.run_import_command(pbf_file)
|
||||
|
|
@ -102,7 +94,7 @@ def test_seconds_to_import_rpi5(size, expected):
|
|||
assert lib.seconds_to_import(size, True) == expected
|
||||
|
||||
|
||||
def test_get_custom_map_directory(test_directory, test_wrolpi_config):
|
||||
def test_get_custom_map_directory(async_client, test_directory, test_wrolpi_config):
|
||||
"""Custom directory can be used for map directory."""
|
||||
# Default location.
|
||||
assert lib.get_map_directory() == (test_directory / 'map')
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from sanic_ext.extensions.openapi import openapi
|
|||
|
||||
from wrolpi.api_utils import json_response
|
||||
from wrolpi.common import logger, wrol_mode_check, get_media_directory, get_relative_to_media_directory
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.schema import JSONErrorResponse
|
||||
from wrolpi.vars import PYTEST
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from wrolpi.errors import APIError, ValidationError, RefreshConflict
|
|||
from wrolpi.vars import PYTEST
|
||||
from .. import schema
|
||||
from ..common import check_for_channel_conflicts
|
||||
from ..errors import UnknownChannel, ChannelDirectoryConflict
|
||||
from ..errors import UnknownChannel
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Channel, Video
|
||||
|
||||
|
|
|
|||
|
|
@ -57,7 +57,8 @@ def test_get_video(test_client, test_session, simple_channel, video_factory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_no_download_frequency(async_client, test_session, test_directory, simple_channel, test_download_manager):
|
||||
async def test_channel_no_download_frequency(async_client, test_session, test_directory, simple_channel,
|
||||
test_download_manager):
|
||||
"""A channel does not require a download frequency."""
|
||||
# No downloads are scheduled.
|
||||
assert len(test_download_manager.get_downloads(test_session)) == 0
|
||||
|
|
@ -465,7 +466,7 @@ async def test_tag_channel(async_client, test_session, test_directory, channel_f
|
|||
v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
|
||||
# Create recurring download which uses the Channel's directory.
|
||||
download = test_download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
|
||||
destination=str(test_directory / 'videos/Channel Name'))
|
||||
destination=str(test_directory / 'videos/Channel Name'))
|
||||
test_session.commit()
|
||||
save_channels_config()
|
||||
assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Channel Name')
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import timedelta
|
||||
|
|
@ -16,7 +14,7 @@ from PIL import Image
|
|||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.captions import FFMPEG_BIN
|
||||
from wrolpi.cmd import FFPROBE_BIN
|
||||
from wrolpi.cmd import FFPROBE_BIN, run_command
|
||||
from wrolpi.common import logger, get_media_directory
|
||||
from wrolpi.dates import seconds_to_timestamp
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
|
|
@ -213,28 +211,23 @@ async def ffprobe_json(video_path: Union[Path, str]) -> dict:
|
|||
],
|
||||
}
|
||||
"""
|
||||
cmd = f'{FFPROBE_BIN}' \
|
||||
f' -print_format json' \
|
||||
f' -loglevel quiet' \
|
||||
f' -show_streams' \
|
||||
f' -show_format' \
|
||||
f' -show_chapters' \
|
||||
f' {shlex.quote(str(video_path.absolute()))}'
|
||||
cmd = (FFPROBE_BIN,
|
||||
'-print_format', 'json',
|
||||
'-loglevel', 'quiet',
|
||||
'-show_streams',
|
||||
'-show_format',
|
||||
'-show_chapters',
|
||||
video_path.absolute())
|
||||
|
||||
result = await run_command(cmd, timeout=60)
|
||||
|
||||
if result.return_code != 0:
|
||||
raise RuntimeError(f'Got non-zero exit code: {result.return_code}')
|
||||
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_shell(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout, stderr = await proc.communicate()
|
||||
content = json.loads(result.stdout.decode().strip())
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to run ffprobe json', exc_info=e)
|
||||
raise
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f'Got non-zero exit code: {proc.returncode}')
|
||||
|
||||
try:
|
||||
content = json.loads(stdout.decode().strip())
|
||||
except Exception as e:
|
||||
logger.debug(stdout.decode())
|
||||
logger.debug(result.stdout.decode())
|
||||
logger.error(f'Failed to load ffprobe json', exc_info=e)
|
||||
raise
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ import json
|
|||
import pathlib
|
||||
import shutil
|
||||
from http import HTTPStatus
|
||||
from itertools import zip_longest
|
||||
from typing import List
|
||||
from uuid import uuid4
|
||||
|
||||
|
|
@ -14,6 +13,7 @@ from modules.videos.downloader import VideoDownloader, ChannelDownloader
|
|||
from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.cmd import CommandResult
|
||||
from wrolpi.downloader import DownloadFrequency, DownloadManager
|
||||
from wrolpi.files.models import FileGroup, Directory
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
|
@ -208,22 +208,11 @@ def mock_video_prepare_filename():
|
|||
@pytest.fixture
|
||||
def mock_video_process_runner():
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_process_runner.return_value = (0, {'stdout': None, 'stderr': None}, None)
|
||||
mock_process_runner.return_value = \
|
||||
CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
yield mock_process_runner
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_video_ids(test_session):
|
||||
"""Check that only the expected Videos are in the DB."""
|
||||
|
||||
def checker(expected_video_ids: List[int]):
|
||||
video_ids = [i.id for i in test_session.query(Video).order_by(Video.id)]
|
||||
for id_, expected in zip_longest(video_ids, expected_video_ids):
|
||||
assert id_ == expected, f'Video ids were not as expected: {expected_video_ids=} != {video_ids=}'
|
||||
|
||||
return checker
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_with_search_factory(test_session, test_directory, video_file_factory):
|
||||
"""A factory that creates a Video record with an associated video File record."""
|
||||
|
|
|
|||
|
|
@ -416,13 +416,12 @@ class VideoDownloader(Downloader, ABC):
|
|||
'-o', video_path,
|
||||
url,
|
||||
)
|
||||
return_code, logs, _ = await self.process_runner(download_id, url, cmd, out_dir)
|
||||
result = await self.process_runner(download, cmd, out_dir)
|
||||
stdout = result.stdout.decode()
|
||||
stderr = result.stderr.decode()
|
||||
|
||||
stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
|
||||
stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
|
||||
|
||||
if return_code != 0:
|
||||
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {return_code}'
|
||||
if result.return_code != 0:
|
||||
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {result.return_code}'
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error=error,
|
||||
|
|
|
|||
|
|
@ -65,19 +65,13 @@ class Video(ModelHelper, Base):
|
|||
except Exception as e:
|
||||
logger.error(f'{self} ffprobe_json is invalid', exc_info=e)
|
||||
|
||||
# TODO these are large objects. Can they be fetched on demand?
|
||||
captions = self.file_group.d_text
|
||||
comments = self.get_comments()
|
||||
|
||||
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
|
||||
d['video'] = dict(
|
||||
caption=captions,
|
||||
caption_files=self.caption_files,
|
||||
channel=channel,
|
||||
channel_id=self.channel_id,
|
||||
codec_names=codec_names,
|
||||
codec_types=codec_types,
|
||||
comments=comments,
|
||||
comments_failed=self.comments_failed,
|
||||
description=self.file_group.c_text or self.get_video_description(),
|
||||
have_comments=self.have_comments,
|
||||
|
|
|
|||
|
|
@ -155,6 +155,16 @@ class VideoResponse:
|
|||
next = ExtendedVideo
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoCommentsResponse:
|
||||
comments: list[dict]
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoCaptionsResponse:
|
||||
captions: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoSearchRequest:
|
||||
search_str: Optional[str] = None
|
||||
|
|
|
|||
|
|
@ -296,10 +296,10 @@ async def test_video_download(test_session, test_directory, mock_video_extract_i
|
|||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
mock_video_process_runner.assert_called_once()
|
||||
download_id, video_url, _, out_dir = mock_video_process_runner.call_args[0]
|
||||
download, cmd, out_dir = mock_video_process_runner.call_args[0]
|
||||
|
||||
download: Download = test_session.query(Download).one()
|
||||
assert video_url == download.url
|
||||
assert download.url in cmd
|
||||
assert test_session.query(Channel).one()
|
||||
|
||||
video: Video = test_session.query(Video).one()
|
||||
|
|
|
|||
|
|
@ -28,6 +28,24 @@ def video_get(_: Request, video_id: int):
|
|||
return json_response({'file_group': video, 'prev': previous_video, 'next': next_video})
|
||||
|
||||
|
||||
@video_bp.get('/video/<video_id:int>/comments')
|
||||
@openapi.description('Get Video comments')
|
||||
@openapi.response(HTTPStatus.OK, schema.VideoCommentsResponse)
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
def video_get_comments(_: Request, video_id: int):
|
||||
video = lib.get_video(video_id)
|
||||
return json_response({'comments': video.get_comments()})
|
||||
|
||||
|
||||
@video_bp.get('/video/<video_id:int>/captions')
|
||||
@openapi.description('Get Video captions')
|
||||
@openapi.response(HTTPStatus.OK, schema.VideoCaptionsResponse)
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
def video_get_captions(_: Request, video_id: int):
|
||||
video = lib.get_video(video_id)
|
||||
return json_response({'captions': video.file_group.d_text})
|
||||
|
||||
|
||||
@video_bp.post('/search')
|
||||
@openapi.definition(
|
||||
summary='Search Video titles and captions',
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
import asyncio
|
||||
import random
|
||||
from datetime import timedelta
|
||||
from typing import Tuple, Optional, List, Dict
|
||||
from typing import Tuple, Optional, List
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.common import run_after, logger, limit_concurrent, wrol_mode_check
|
||||
from wrolpi.common import logger, limit_concurrent, wrol_mode_check
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, optional_session
|
||||
from wrolpi.downloader import download_manager
|
||||
|
|
@ -17,7 +17,6 @@ from wrolpi.files.models import FileGroup
|
|||
from wrolpi.tags import tag_append_sub_select_where
|
||||
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR
|
||||
from ..errors import UnknownVideo
|
||||
from ..lib import save_channels_config
|
||||
|
||||
logger.getChild(__name__)
|
||||
|
||||
|
|
@ -38,6 +37,15 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
|
|||
return video, previous_video, next_video
|
||||
|
||||
|
||||
def get_video(video_id: int) -> Video:
|
||||
"""
|
||||
Get a Video, with it's prev/next videos. Mark the Video as viewed.
|
||||
"""
|
||||
with get_db_session() as session:
|
||||
video = Video.find_by_id(video_id, session=session)
|
||||
return video
|
||||
|
||||
|
||||
VIDEO_ORDERS = {
|
||||
'published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) ASC, LOWER(fg.primary_path) ASC',
|
||||
'-published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) DESC NULLS LAST,'
|
||||
|
|
|
|||
|
|
@ -187,3 +187,23 @@ async def test_wrol_mode(async_client, simple_channel, simple_video, wrol_mode_f
|
|||
|
||||
await wrol_mode_fixture(False)
|
||||
assert not test_download_manager.stopped.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_video_extras(async_client, simple_channel, video_factory):
|
||||
"""Can fetch extra data about a video (comments/captions)."""
|
||||
info_json = {'duration': 5, 'epoch': 12345, 'comments': [{'some': 'comments'}]}
|
||||
video = video_factory(simple_channel.id, with_info_json=info_json, with_caption_file=True)
|
||||
|
||||
request, response = await async_client.get(f'/api/videos/video/{video.id}/captions')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json.get('captions')
|
||||
|
||||
request, response = await async_client.get(f'/api/videos/video/{video.id}/comments')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json.get('comments')
|
||||
|
||||
request, response = await async_client.get('/api/videos/video/123/captions')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
request, response = await async_client.get('/api/videos/video/123/comments')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ class KiwixZimDownloader(Downloader):
|
|||
zim_directory = lib.get_zim_directory()
|
||||
zim_directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
output_path = await self.download_file(download.id, download.url, zim_directory)
|
||||
output_path = await self.download_file(download, download.url, zim_directory)
|
||||
|
||||
# Notify the maintainer if outdated Zim files are lying around.
|
||||
lib.flag_outdated_zim_files()
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from modules.zim.errors import UnknownZim, UnknownZimSubscription
|
|||
from modules.zim.kiwix import KIWIX_CATALOG
|
||||
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
|
||||
from wrolpi import flags
|
||||
from wrolpi.cmd import run_command
|
||||
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
|
||||
register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate
|
||||
from wrolpi.db import get_db_session, optional_session, get_db_curs
|
||||
|
|
@ -471,13 +472,12 @@ async def check_zim(path: pathlib.Path) -> int:
|
|||
|
||||
@warning: Only performs the checksum check!
|
||||
"""
|
||||
cmd = f'zimcheck -C {path.absolute()}'
|
||||
proc = await asyncio.create_subprocess_shell(cmd)
|
||||
stdout, stderr = await proc.communicate()
|
||||
logger.debug(f'zimcheck returned {proc.returncode}')
|
||||
if proc.returncode != 0 and stderr:
|
||||
logger.debug(stderr)
|
||||
return proc.returncode
|
||||
cmd = ['zimcheck', '-C', path.absolute()]
|
||||
result = await run_command(cmd)
|
||||
logger.debug(f'zimcheck returned {result.return_code}')
|
||||
if result.return_code != 0 and result.stderr:
|
||||
logger.debug(result.stderr)
|
||||
return result.return_code
|
||||
|
||||
|
||||
ZIM_NAME_PARSER = re.compile(r'(.+?)_(\d{4}-\d{2}).zim')
|
||||
|
|
@ -566,10 +566,9 @@ async def restart_kiwix():
|
|||
|
||||
logger.info('Restarting Kiwix serve')
|
||||
|
||||
cmd = f'sudo /usr/bin/systemctl restart wrolpi-kiwix.service'
|
||||
proc = await asyncio.create_subprocess_shell(cmd)
|
||||
stdout, stderr = await proc.communicate()
|
||||
logger.debug(f'systemctl returned {proc.returncode}')
|
||||
if proc.returncode != 0 and stderr:
|
||||
logger.debug(stderr)
|
||||
return proc.returncode
|
||||
cmd = ('sudo', '/usr/bin/systemctl', 'restart' 'wrolpi-kiwix.service')
|
||||
result = await run_command(cmd)
|
||||
logger.debug(f'systemctl returned {result.return_code}')
|
||||
if result.return_code != 0 and result.stderr:
|
||||
logger.debug(result.stderr)
|
||||
return result.return_code
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from wrolpi.common import DownloadFileInfo, get_wrolpi_config
|
|||
from wrolpi.downloader import Download, import_downloads_config, save_downloads_config
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.test.common import skip_circleci
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -182,6 +183,7 @@ def test_zim_all_entries(test_session, test_zim):
|
|||
assert len(entries) == 12
|
||||
|
||||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock,
|
||||
await_switches):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue