Creating new /api/archive/upload which accepts a file from the SingleFile browser extension and converts it to an Archive.

This commit is contained in:
Roland 2025-01-26 12:44:16 -07:00
parent 01338e4d89
commit 531ec3d65e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
62 changed files with 1000 additions and 562 deletions

View file

@ -1,18 +1,15 @@
import asyncio
import json
import pathlib
import tempfile
from abc import ABC
from json import JSONDecodeError
from typing import List, Tuple, Iterable
from selenium import webdriver
from sqlalchemy import not_
from sqlalchemy.orm import Session
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM
from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
slow_logger, html_screenshot, get_title_from_html
slow_logger, get_title_from_html
from wrolpi.db import optional_session, get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
@ -38,6 +35,11 @@ class ArchiveDownloader(Downloader, ABC):
def __repr__(self):
return f'<ArchiveDownloader>'
@optional_session
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
return file_groups
async def do_download(self, download: Download) -> DownloadResult:
if download.attempts > 3:
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
@ -45,12 +47,13 @@ class ArchiveDownloader(Downloader, ABC):
if DOCKERIZED or PYTEST:
# Perform the archive in the Archive docker container. (Typically in the development environment).
singlefile, readability, screenshot = await request_archive(download.url)
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
archive_id = archive.id
else:
# Perform the archive using locally installed executables.
singlefile, readability, screenshot = await self.do_archive(download)
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
archive_id = archive.id
singlefile = await self.do_singlefile(download)
archive = await lib.singlefile_to_archive(singlefile)
archive_id = archive.id
with get_db_session() as session:
archive = Archive.find_by_id(archive_id, session)
@ -67,12 +70,7 @@ class ArchiveDownloader(Downloader, ABC):
return DownloadResult(success=True, location=f'/archive/{archive.id}')
@optional_session
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
return file_groups
async def do_singlefile(self, download: Download) -> Tuple[bytes, dict]:
async def do_singlefile(self, download: Download) -> bytes:
"""Create a Singlefile from the archive's URL."""
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
str(SINGLE_FILE_BIN),
@ -83,115 +81,26 @@ class ArchiveDownloader(Downloader, ABC):
'--load-deferred-images-dispatch-scroll-event',
download.url,
)
return_code, logs, stdout = await self.process_runner(
download.id,
download.url,
cmd,
pathlib.Path('/home/wrolpi'),
)
cwd = pathlib.Path('/home/wrolpi')
cwd = cwd if cwd.is_dir() else None
result = await self.process_runner(download, cmd, cwd)
stderr = logs['stderr'].decode()
log_output = stderr or stdout.decode() or 'No stderr or stdout!'
stderr = result.stderr.decode()
log_output = stderr or result.stdout.decode() or 'No stderr or stdout!'
if return_code != 0:
if result.return_code != 0:
e = ChildProcessError(log_output[:1000])
raise RuntimeError(f'singlefile exited with {return_code}') from e
raise RuntimeError(f'singlefile exited with {result.return_code}') from e
if not stdout:
if not result.stdout:
e = ChildProcessError(log_output[:1000])
raise RuntimeError(f'Singlefile created was empty: {download.url}') from e
if SINGLEFILE_HEADER.encode() not in stdout:
if SINGLEFILE_HEADER.encode() not in result.stdout:
e = ChildProcessError(log_output[:1000])
raise RuntimeError(f'Singlefile created was invalid: {download.url}') from e
return stdout, logs
async def do_readability(self, download: Download, html: bytes) -> dict:
"""Extract the readability dict from the provided HTML."""
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
fh.write(html)
cmd = (READABILITY_BIN, fh.name, download.url)
logger.debug(f'readability cmd: {cmd}')
return_code, logs, stdout = await self.process_runner(
download.id,
download.url,
cmd,
pathlib.Path('/home/wrolpi'),
)
if return_code == 0:
if not stdout:
raise RuntimeError('readability stdout was empty')
try:
readability = json.loads(stdout)
except TypeError or JSONDecodeError:
# JSON was invalid.
e = ChildProcessError(stdout.decode() if stdout else 'No stdout')
if logs and (stderr := logs.get('stderr')):
e = ChildProcessError(stderr.decode())
raise RuntimeError(f'Failed to extract readability from {download.url}') from e
logger.debug(f'done readability for {download.url}')
return readability
else:
logger.error(f'Failed to extract readability for {download.url}')
e = ChildProcessError(stdout.decode() if stdout else 'No stdout')
if logs and (stderr := logs.get('stderr')):
e = ChildProcessError(stderr.decode())
raise RuntimeError(f'Failed to extract readability for {download.url} got {return_code}') from e
@staticmethod
async def do_screenshot_url(download: Download) -> bytes:
"""Use Chromium to get a screenshot of the Download's URL."""
# Set Chromium to headless. Use a wide window size so that screenshot will be the "desktop" version of
# the page.
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument('disable-gpu')
options.add_argument('window-size=1280x720')
driver = webdriver.Chrome(chrome_options=options)
driver.get(download.url)
screenshot = driver.get_screenshot_as_png()
return screenshot
async def do_archive(self, download: Download) -> Tuple[bytes, dict, bytes]:
"""Use locally installed executables to create an Archive.
Creates a Singlefile, readability file, and screenshot file.
@warning: Will not raise errors if readability or screenshot cannot be extracted.
"""
singlefile, logs = await self.do_singlefile(download)
# Extract Readability from the Singlefile.
try:
readability = await self.do_readability(download, singlefile)
except RuntimeError:
# Readability is not required.
readability = None
screenshot = b''
try:
# Screenshot the Singlefile first.
screenshot = html_screenshot(singlefile)
except Exception as e:
logger.error(f'Failed to screenshot file for {download.url}', exc_info=e)
if not screenshot:
# Screenshot of singlefile failed. Download the URL again.
try:
screenshot = await self.do_screenshot_url(download)
except Exception as e:
# Screenshot failed.
logger.error(f'Failed to screenshot {download.url}', exc_info=e)
singlefile_len = len(singlefile) if singlefile else None
readability_len = len(readability) if readability else None
screenshot_len = len(screenshot) if screenshot else None
logger.debug(f'do_archive of {download.url} finished: {singlefile_len=} {readability_len=} {screenshot_len=}')
return singlefile, readability, screenshot
return result.stdout
archive_downloader = ArchiveDownloader()
@ -258,7 +167,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
archive.validate()
archive.flush()
file_group.title = file_group.a_text = title
file_group.title = file_group.a_text = title or archive.file_group.title
file_group.d_text = contents
file_group.data = {
'id': archive.id,

View file

@ -7,7 +7,9 @@ from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger, wrol_mode_check, api_param_limiter
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
from wrolpi.switches import register_switch_handler
from . import lib, schema
NAME = 'archive'
@ -29,7 +31,7 @@ async def get_archive(_: Request, archive_id: int):
@archive_bp.delete('/<archive_ids:int>', name='archive_delete_one')
@archive_bp.delete('/<archive_ids:[0-9,]+>', name='archive_delete_many')
@openapi.description('Delete an individual archive')
@openapi.description('Delete Archives')
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@wrol_mode_check
async def delete_archive(_: Request, archive_ids: str):
@ -70,3 +72,38 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
body.headline)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)
@register_switch_handler('singlefile_upload_switch_handler')
async def singlefile_upload_switch_handler(singlefile):
archive = await lib.singlefile_to_archive(singlefile)
logger.info(f'Created Archive from upload: {archive}')
name = archive.file_group.title or archive.file_group.url
Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location)
@archive_bp.get('/upload')
@openapi.definition(
summary='A message to confirm to the user that they have the correct upload URL.'
)
async def get_upload_singlefile(request: Request):
return response.text('This is the URL to upload files using the SingleFile browser extension.'
' This requires a POST request.')
@archive_bp.post('/upload')
@openapi.definition(
summary='Upload SingleFile from SingleFile browser extension and convert it to an Archive.'
)
async def post_upload_singlefile(request: Request):
url = request.form['url'][0]
singlefile = request.files['singlefile_contents'][0].body
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url}')
# Get the URL to ensure that the singlefile is valid.
lib.get_url_from_singlefile(singlefile)
# Send processing to background task so the extension can continue.
singlefile_upload_switch_handler.activate_switch(dict(singlefile=singlefile))
# Return empty json response because SingleFile extension expects a JSON response.
return json_response(dict(), status=HTTPStatus.OK)

View file

@ -1,10 +1,15 @@
import base64
import base64
import gzip
import json
import os
import pathlib
import re
import shlex
import tempfile
from dataclasses import dataclass
from datetime import datetime
from json import JSONDecodeError
from typing import Optional, Tuple, List, Union
from urllib.parse import urlparse
@ -14,13 +19,14 @@ from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from wrolpi import dates
from wrolpi.cmd import READABILITY_BIN, run_command
from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config, html_screenshot
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
from wrolpi.tags import tag_append_sub_select_where
from wrolpi.vars import PYTEST
from wrolpi.vars import PYTEST, DOCKERIZED
logger = logger.getChild(__name__)
@ -106,11 +112,11 @@ def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles:
ARCHIVE_TIMEOUT = Seconds.minute * 10 # Wait at most 10 minutes for response.
async def request_archive(url: str) -> Tuple[str, Optional[str], Optional[str]]:
async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Optional[dict], Optional[str]]:
"""Send a request to the archive service to archive the URL."""
logger.info(f'Sending archive request to archive service: {url}')
data = {'url': url}
data = dict(url=url, singlefile=singlefile)
try:
async with aiohttp_post(f'{ARCHIVE_SERVICE}/json', json_=data, timeout=ARCHIVE_TIMEOUT) as response:
status = response.status
@ -229,9 +235,9 @@ def get_or_create_domain(session: Session, url) -> Domain:
SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n')
def get_url_from_singlefile(html: bytes) -> str:
def get_url_from_singlefile(html: bytes | str) -> str:
"""Extract URL from SingleFile contents."""
html = html.decode()
html = html.decode() if isinstance(html, bytes) else html
if SINGLEFILE_HEADER not in html:
raise RuntimeError(f'Not a singlefile!')
@ -313,7 +319,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
except json.decoder.JSONDecodeError:
# Was not valid JSON.
schema = None
if isinstance(schema, dict) and schema.get('@context') in ('https://schema.org', 'http://schema.org'):
if isinstance(schema, dict) and '://schema.org' in schema.get('@context'):
# Found https://schema.org/
if headline := schema.get('headline'):
metadata.title = metadata.title or headline
@ -347,6 +353,17 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
metadata.author = author
else:
logger.warning(f'Unable to parse author schema: {author}')
elif authors := schema.get('authors'):
# Use the first Author.
author = authors[0]
if isinstance(author, dict):
author = author.get('name') or author
if isinstance(author, str):
metadata.author = author
else:
logger.warning(f'Unable to parse author schema: {author}')
# Assume UTC if no timezone.
if metadata.published_datetime and not metadata.published_datetime.tzinfo and assume_utc:
@ -368,7 +385,7 @@ def get_archive(session, archive_id: int) -> Archive:
def delete_archives(*archive_ids: List[int]):
"""Delete an Archive and all of it's files."""
"""Delete an Archive and all of its files."""
with get_db_session(commit=True) as session:
archives: List[Archive] = list(session.query(Archive).filter(Archive.id.in_(archive_ids)))
if not archives:
@ -535,3 +552,75 @@ async def search_domains_by_name(name: str, limit: int = 5, session: Session = N
.limit(limit) \
.all()
return domains
async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
"""Extract the readability dict from the provided HTML."""
with tempfile.NamedTemporaryFile('wb', suffix='.html') as singlefile_file:
singlefile_file.write(html.encode() if isinstance(html, str) else html)
singlefile_file.flush()
os.fsync(singlefile_file.fileno())
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
READABILITY_BIN, singlefile_file.name, shlex.quote(url))
# Docker containers may not have this directory. But, this directory is necessary on RPi.
cwd = '/home/wrolpi' if os.path.isdir('/home/wrolpi') else None
result = await run_command(cmd, cwd=cwd, timeout=timeout, debug=True)
logger.debug(f'readability for {url} exited with {result.return_code}')
stdout, stderr = result.stdout.decode(), result.stderr.decode()
if result.return_code == 0:
if not stdout:
raise RuntimeError('readability stdout was empty')
try:
readability = json.loads(stdout)
except TypeError or JSONDecodeError:
# JSON was invalid.
e = ChildProcessError(stdout if stdout else 'No stdout')
if stderr:
e = ChildProcessError(stderr)
raise RuntimeError(f'Failed to extract readability from {url}') from e
logger.debug(f'done readability for {url}')
return readability
else:
logger.error(f'Failed to extract readability for {url}')
e = ChildProcessError(stdout if stdout else 'No stdout')
if stderr:
e = ChildProcessError(stderr)
raise RuntimeError(f'Failed to extract readability for {url} got {result.return_code}') from e
async def singlefile_to_archive(singlefile: bytes) -> Archive:
"""
Convert a SingleFile to an Archive.
This is done by extracting readability, creating a screenshot, then attaching them to an Archive/FileGroup.
"""
# Get URL first because it does some simple checking of `singlefile`
url = get_url_from_singlefile(singlefile)
singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile
if DOCKERIZED:
# Perform the archive in the Archive docker container. (Typically in the development environment).
singlefile = base64.b64encode(singlefile).decode()
logger.debug(f'singlefile_to_archive sending to archive service: {url}')
singlefile, readability, screenshot = await request_archive(url, singlefile=singlefile)
else:
# JSON from readability-extractor
readability = dict()
try:
logger.debug(f'singlefile_to_archive extracting readability: {url}')
readability = await html_to_readability(singlefile, url)
except RuntimeError as e:
logger.error(f'Failed to extract readability from: {url}', exc_info=e)
screenshot = None
try:
logger.debug(f'singlefile_to_archive creating screenshot: {url}')
screenshot = html_screenshot(singlefile)
except Exception as e:
logger.error(f'Failed to extract screenshot from: {url}', exc_info=e)
logger.trace(f'singlefile_to_archive modeling: {url}')
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot)
return archive

View file

@ -318,6 +318,11 @@ class Archive(Base, ModelHelper):
def add_tag(self, tag_id_or_name: int | str, session: Session = None) -> TagFile:
return self.file_group.add_tag(tag_id_or_name, session)
@property
def location(self):
"""The location where this Archive can be viewed in the UI."""
return f'/archive/{self.id}'
class Domain(Base, ModelHelper):
__tablename__ = 'domains' # plural to avoid conflict

View file

@ -3,7 +3,8 @@ from http import HTTPStatus
import pytest
from modules.archive import lib
from modules.archive import lib, Archive
from wrolpi.test.common import skip_circleci
def check_results(test_client, data, ids):
@ -227,3 +228,31 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
request, response = test_client.get(f'/api/archive/domains')
assert response.status_code == HTTPStatus.OK
assert response.json['domains'] == []
@skip_circleci
@pytest.mark.asyncio
async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form,
await_switches, events_history):
"""Test converting a SingleFile from the SingleFile browser extension to an Archive."""
singlefile_contents = singlefile_contents_factory(title='upload title', url='https://example.com/single-file-url')
forms = [
dict(name='url', value='https://example.com/form-url'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents')
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
await await_switches()
archive, = test_session.query(Archive).all()
assert archive.singlefile_path.is_file(), 'Singlefile should have been saved.'
assert archive.readability_path.is_file(), 'Readability should be extracted.'
assert archive.readability_json_path.is_file(), 'Readability json should be extracted.'
assert archive.screenshot_path.is_file(), 'Screenshot should have been created'
assert archive.file_group.title == 'upload title', 'Title was not correct'
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
events = list(events_history)
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'

View file

@ -561,6 +561,35 @@ async def test_refresh_archives_index(test_session, make_files_structure):
assert archive.file_group.indexed is True
@pytest.mark.asyncio
async def test_archive_meta(async_client, test_session, make_files_structure):
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en">
<script data-vue-meta="ssr" type="application/ld+json">
{"@context":"https://schema.org/","@type":"NewsArticle","headline":"The Headline","description":"The Description","authors":[{"name":"A.B.C.","@type":"Person"}],"datePublished":"2021-06-20T10:00"}
</script>
</html>'''
singlefile, *_ = make_files_structure({
'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents,
'archive/example.com/2021-10-05-16-20-10_NA.png': None,
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents',
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
})
await files_lib.refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
assert archive.file_group.author == 'A.B.C.'
assert archive.file_group.published_datetime == datetime(2021, 6, 20, 10, 0, tzinfo=pytz.utc)
assert archive.file_group.title == 'The Headline'
assert archive.file_group.b_text == 'The Description'
assert archive.file_group.indexed is True
@pytest.mark.asyncio
async def test_refresh_archives_deleted_singlefile(test_session, make_files_structure, singlefile_contents_factory):
"""Removing a Singlefile file from a FileGroup makes that group no longer an Archive."""

View file

@ -1,4 +1,3 @@
import wrolpi
from wrolpi.db import get_db_session
from .api import inventory_bp
from .inventory import logger, DEFAULT_CATEGORIES, DEFAULT_INVENTORIES

View file

@ -1,5 +1,5 @@
import asyncio
import subprocess
from datetime import timedelta
from pathlib import Path
from typing import List
@ -8,9 +8,9 @@ from sqlalchemy.orm import Session
from modules.map.models import MapFile
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import SUDO_BIN
from wrolpi.cmd import SUDO_BIN, run_command
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config
from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp
from wrolpi.dates import timedelta_to_timestamp, seconds_to_timestamp
from wrolpi.db import optional_session, get_db_session
from wrolpi.events import Events
from wrolpi.vars import PROJECT_DIR, IS_RPI5
@ -37,7 +37,7 @@ def is_pbf_file(pbf: Path) -> bool:
def is_dump_file(path: Path) -> bool:
"""Uses file command to check type of a file. Returns True if a file is a Postgresql dump file."""
"""Uses file command to check the type of file. Returns True if a file is a Postgresql dump file."""
cmd = ('/usr/bin/file', path)
try:
output = subprocess.check_output(cmd)
@ -188,24 +188,22 @@ async def run_import_command(*paths: Path) -> int:
paths = ' '.join(str(i) for i in paths)
# Run with sudo so renderd can be restarted.
cmd = f'{SUDO_BIN} {PROJECT_DIR}/scripts/import_map.sh {paths}'
cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', paths)
import_logger.warning(f'Running map import command: {cmd}')
start = now()
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
result = await run_command(cmd, debug=True)
stdout, stderr = await proc.communicate()
elapsed = now() - start
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
import_logger.info(f'{success} import of {repr(paths)} took {timedelta_to_timestamp(elapsed)}')
if proc.returncode != 0:
success = 'Successful' if result.return_code == 0 else 'Unsuccessful'
import_logger.info(
f'{success} import of {repr(paths)} took {timedelta_to_timestamp(timedelta(seconds=result.elapsed))}')
if result.return_code != 0:
# Log all lines. Truncate long lines.
for line in stdout.decode().splitlines():
for line in result.stdout.decode().splitlines():
import_logger.debug(line[:500])
for line in stderr.decode().splitlines():
for line in result.stderr.decode().splitlines():
import_logger.error(line[:500])
raise ValueError(f'Importing map file failed with return code {proc.returncode}')
raise ValueError(f'Importing map file failed with return code {result.return_code}')
return int(elapsed.total_seconds())
return result.elapsed
@optional_session

View file

@ -16,7 +16,7 @@ def assert_map_file_status(test_client, expected, status_code=HTTPStatus.OK):
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_status_and_import(test_client, test_session, make_files_structure, mock_create_subprocess_shell):
def test_status_and_import(test_client, test_session, make_files_structure, mock_run_command):
"""PBF files can be imported, and the status of the import can be monitored."""
pbf1, pbf2, dump1 = make_files_structure([
'map/pbf/some-country.osm.pbf',
@ -32,11 +32,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf1), ], }
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf1), ], }
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
@ -45,11 +43,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
@ -58,11 +54,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(dump1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(dump1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': True, 'path': 'map/dump/dumpy.dump'},
@ -74,7 +68,7 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_create_subprocess_shell):
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_run_command):
"""Multiple PBFs can be imported. Importing a second time overwrites the previous imports."""
pbf1, pbf2, pbf3, dump = make_files_structure([
'map/pbf/country1.osm.pbf',
@ -83,11 +77,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
'map/dumpy.dump',
])
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf2), str(pbf1)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf2), str(pbf1)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Two PBFs can be merged and imported.
expected = [
@ -98,11 +90,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf3), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf3), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# New imports overwrite old imports.
expected = [
@ -113,12 +103,10 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
# Can't mix PBF and dump.
body = {'files': [str(dump), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Can't mix PBF and dump.
body = {'files': [str(dump), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Dump does not overwrite.
expected = [

View file

@ -22,7 +22,7 @@ def test_get_map_paths(test_directory, make_files_structure):
@pytest.mark.asyncio
async def test_run_import_command(test_directory, mock_create_subprocess_shell):
async def test_run_import_command(test_directory, mock_run_command, run_command_bad_result):
"""
Import map files. Files to import are checked for validity before importing. Any errors returned by
asyncio.create_subprocess_shell are caught and reported.
@ -38,22 +38,14 @@ async def test_run_import_command(test_directory, mock_create_subprocess_shell):
dump_file = test_directory / 'dumpy.dump'
dump_file.touch()
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
# Run the import, it succeeds.
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
communicate_return=(b'out', b'error')
)
mock_is_pbf_file.return_value = True
await lib.run_import_command(pbf_file)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
# Run the import, it fails.
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
communicate_return=(b'out', b'error'),
return_code=1,
)
mock_run_command(run_command_bad_result)
mock_is_pbf_file.return_value = True
with pytest.raises(ValueError):
await lib.run_import_command(pbf_file)
@ -102,7 +94,7 @@ def test_seconds_to_import_rpi5(size, expected):
assert lib.seconds_to_import(size, True) == expected
def test_get_custom_map_directory(test_directory, test_wrolpi_config):
def test_get_custom_map_directory(async_client, test_directory, test_wrolpi_config):
"""Custom directory can be used for map directory."""
# Default location.
assert lib.get_map_directory() == (test_directory / 'map')

View file

@ -9,7 +9,6 @@ from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger, wrol_mode_check, get_media_directory, get_relative_to_media_directory
from wrolpi.downloader import download_manager
from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
from wrolpi.vars import PYTEST

View file

@ -14,7 +14,7 @@ from wrolpi.errors import APIError, ValidationError, RefreshConflict
from wrolpi.vars import PYTEST
from .. import schema
from ..common import check_for_channel_conflicts
from ..errors import UnknownChannel, ChannelDirectoryConflict
from ..errors import UnknownChannel
from ..lib import save_channels_config
from ..models import Channel, Video

View file

@ -57,7 +57,8 @@ def test_get_video(test_client, test_session, simple_channel, video_factory):
@pytest.mark.asyncio
async def test_channel_no_download_frequency(async_client, test_session, test_directory, simple_channel, test_download_manager):
async def test_channel_no_download_frequency(async_client, test_session, test_directory, simple_channel,
test_download_manager):
"""A channel does not require a download frequency."""
# No downloads are scheduled.
assert len(test_download_manager.get_downloads(test_session)) == 0
@ -465,7 +466,7 @@ async def test_tag_channel(async_client, test_session, test_directory, channel_f
v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
# Create recurring download which uses the Channel's directory.
download = test_download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
destination=str(test_directory / 'videos/Channel Name'))
destination=str(test_directory / 'videos/Channel Name'))
test_session.commit()
save_channels_config()
assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Channel Name')

View file

@ -1,9 +1,7 @@
import asyncio
import json
import os
import pathlib
import re
import shlex
import subprocess
import tempfile
from datetime import timedelta
@ -16,7 +14,7 @@ from PIL import Image
from sqlalchemy.orm import Session
from wrolpi.captions import FFMPEG_BIN
from wrolpi.cmd import FFPROBE_BIN
from wrolpi.cmd import FFPROBE_BIN, run_command
from wrolpi.common import logger, get_media_directory
from wrolpi.dates import seconds_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
@ -213,28 +211,23 @@ async def ffprobe_json(video_path: Union[Path, str]) -> dict:
],
}
"""
cmd = f'{FFPROBE_BIN}' \
f' -print_format json' \
f' -loglevel quiet' \
f' -show_streams' \
f' -show_format' \
f' -show_chapters' \
f' {shlex.quote(str(video_path.absolute()))}'
cmd = (FFPROBE_BIN,
'-print_format', 'json',
'-loglevel', 'quiet',
'-show_streams',
'-show_format',
'-show_chapters',
video_path.absolute())
result = await run_command(cmd, timeout=60)
if result.return_code != 0:
raise RuntimeError(f'Got non-zero exit code: {result.return_code}')
try:
proc = await asyncio.create_subprocess_shell(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = await proc.communicate()
content = json.loads(result.stdout.decode().strip())
except Exception as e:
logger.error(f'Failed to run ffprobe json', exc_info=e)
raise
if proc.returncode != 0:
raise RuntimeError(f'Got non-zero exit code: {proc.returncode}')
try:
content = json.loads(stdout.decode().strip())
except Exception as e:
logger.debug(stdout.decode())
logger.debug(result.stdout.decode())
logger.error(f'Failed to load ffprobe json', exc_info=e)
raise

View file

@ -2,7 +2,6 @@ import json
import pathlib
import shutil
from http import HTTPStatus
from itertools import zip_longest
from typing import List
from uuid import uuid4
@ -14,6 +13,7 @@ from modules.videos.downloader import VideoDownloader, ChannelDownloader
from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination
from modules.videos.models import Channel, Video
from wrolpi.api_utils import api_app
from wrolpi.cmd import CommandResult
from wrolpi.downloader import DownloadFrequency, DownloadManager
from wrolpi.files.models import FileGroup, Directory
from wrolpi.vars import PROJECT_DIR
@ -208,22 +208,11 @@ def mock_video_prepare_filename():
@pytest.fixture
def mock_video_process_runner():
with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_process_runner.return_value = (0, {'stdout': None, 'stderr': None}, None)
mock_process_runner.return_value = \
CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
yield mock_process_runner
@pytest.fixture
def assert_video_ids(test_session):
"""Check that only the expected Videos are in the DB."""
def checker(expected_video_ids: List[int]):
video_ids = [i.id for i in test_session.query(Video).order_by(Video.id)]
for id_, expected in zip_longest(video_ids, expected_video_ids):
assert id_ == expected, f'Video ids were not as expected: {expected_video_ids=} != {video_ids=}'
return checker
@pytest.fixture
def video_with_search_factory(test_session, test_directory, video_file_factory):
"""A factory that creates a Video record with an associated video File record."""

View file

@ -416,13 +416,12 @@ class VideoDownloader(Downloader, ABC):
'-o', video_path,
url,
)
return_code, logs, _ = await self.process_runner(download_id, url, cmd, out_dir)
result = await self.process_runner(download, cmd, out_dir)
stdout = result.stdout.decode()
stderr = result.stderr.decode()
stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
if return_code != 0:
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {return_code}'
if result.return_code != 0:
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {result.return_code}'
return DownloadResult(
success=False,
error=error,

View file

@ -65,19 +65,13 @@ class Video(ModelHelper, Base):
except Exception as e:
logger.error(f'{self} ffprobe_json is invalid', exc_info=e)
# TODO these are large objects. Can they be fetched on demand?
captions = self.file_group.d_text
comments = self.get_comments()
# Put live data in "video" instead of "data" to avoid confusion on the frontend.
d['video'] = dict(
caption=captions,
caption_files=self.caption_files,
channel=channel,
channel_id=self.channel_id,
codec_names=codec_names,
codec_types=codec_types,
comments=comments,
comments_failed=self.comments_failed,
description=self.file_group.c_text or self.get_video_description(),
have_comments=self.have_comments,

View file

@ -155,6 +155,16 @@ class VideoResponse:
next = ExtendedVideo
@dataclass
class VideoCommentsResponse:
comments: list[dict]
@dataclass
class VideoCaptionsResponse:
captions: str
@dataclass
class VideoSearchRequest:
search_str: Optional[str] = None

View file

@ -296,10 +296,10 @@ async def test_video_download(test_session, test_directory, mock_video_extract_i
await video_download_manager.wait_for_all_downloads()
mock_video_process_runner.assert_called_once()
download_id, video_url, _, out_dir = mock_video_process_runner.call_args[0]
download, cmd, out_dir = mock_video_process_runner.call_args[0]
download: Download = test_session.query(Download).one()
assert video_url == download.url
assert download.url in cmd
assert test_session.query(Channel).one()
video: Video = test_session.query(Video).one()

View file

@ -28,6 +28,24 @@ def video_get(_: Request, video_id: int):
return json_response({'file_group': video, 'prev': previous_video, 'next': next_video})
@video_bp.get('/video/<video_id:int>/comments')
@openapi.description('Get Video comments')
@openapi.response(HTTPStatus.OK, schema.VideoCommentsResponse)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
def video_get_comments(_: Request, video_id: int):
video = lib.get_video(video_id)
return json_response({'comments': video.get_comments()})
@video_bp.get('/video/<video_id:int>/captions')
@openapi.description('Get Video captions')
@openapi.response(HTTPStatus.OK, schema.VideoCaptionsResponse)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
def video_get_captions(_: Request, video_id: int):
video = lib.get_video(video_id)
return json_response({'captions': video.file_group.d_text})
@video_bp.post('/search')
@openapi.definition(
summary='Search Video titles and captions',

View file

@ -1,14 +1,14 @@
import asyncio
import random
from datetime import timedelta
from typing import Tuple, Optional, List, Dict
from typing import Tuple, Optional, List
import yt_dlp
from sqlalchemy import or_
from sqlalchemy.orm import Session
from modules.videos.models import Video
from wrolpi.common import run_after, logger, limit_concurrent, wrol_mode_check
from wrolpi.common import logger, limit_concurrent, wrol_mode_check
from wrolpi.dates import now
from wrolpi.db import get_db_session, optional_session
from wrolpi.downloader import download_manager
@ -17,7 +17,6 @@ from wrolpi.files.models import FileGroup
from wrolpi.tags import tag_append_sub_select_where
from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR
from ..errors import UnknownVideo
from ..lib import save_channels_config
logger.getChild(__name__)
@ -38,6 +37,15 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
return video, previous_video, next_video
def get_video(video_id: int) -> Video:
"""
Get a Video, with it's prev/next videos. Mark the Video as viewed.
"""
with get_db_session() as session:
video = Video.find_by_id(video_id, session=session)
return video
VIDEO_ORDERS = {
'published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) ASC, LOWER(fg.primary_path) ASC',
'-published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) DESC NULLS LAST,'

View file

@ -187,3 +187,23 @@ async def test_wrol_mode(async_client, simple_channel, simple_video, wrol_mode_f
await wrol_mode_fixture(False)
assert not test_download_manager.stopped.is_set()
@pytest.mark.asyncio
async def test_api_video_extras(async_client, simple_channel, video_factory):
"""Can fetch extra data about a video (comments/captions)."""
info_json = {'duration': 5, 'epoch': 12345, 'comments': [{'some': 'comments'}]}
video = video_factory(simple_channel.id, with_info_json=info_json, with_caption_file=True)
request, response = await async_client.get(f'/api/videos/video/{video.id}/captions')
assert response.status_code == HTTPStatus.OK
assert response.json.get('captions')
request, response = await async_client.get(f'/api/videos/video/{video.id}/comments')
assert response.status_code == HTTPStatus.OK
assert response.json.get('comments')
request, response = await async_client.get('/api/videos/video/123/captions')
assert response.status_code == HTTPStatus.NOT_FOUND
request, response = await async_client.get('/api/videos/video/123/comments')
assert response.status_code == HTTPStatus.NOT_FOUND

View file

@ -74,7 +74,7 @@ class KiwixZimDownloader(Downloader):
zim_directory = lib.get_zim_directory()
zim_directory.mkdir(parents=True, exist_ok=True)
output_path = await self.download_file(download.id, download.url, zim_directory)
output_path = await self.download_file(download, download.url, zim_directory)
# Notify the maintainer if outdated Zim files are lying around.
lib.flag_outdated_zim_files()

View file

@ -15,6 +15,7 @@ from modules.zim.errors import UnknownZim, UnknownZimSubscription
from modules.zim.kiwix import KIWIX_CATALOG
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
from wrolpi import flags
from wrolpi.cmd import run_command
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate
from wrolpi.db import get_db_session, optional_session, get_db_curs
@ -471,13 +472,12 @@ async def check_zim(path: pathlib.Path) -> int:
@warning: Only performs the checksum check!
"""
cmd = f'zimcheck -C {path.absolute()}'
proc = await asyncio.create_subprocess_shell(cmd)
stdout, stderr = await proc.communicate()
logger.debug(f'zimcheck returned {proc.returncode}')
if proc.returncode != 0 and stderr:
logger.debug(stderr)
return proc.returncode
cmd = ['zimcheck', '-C', path.absolute()]
result = await run_command(cmd)
logger.debug(f'zimcheck returned {result.return_code}')
if result.return_code != 0 and result.stderr:
logger.debug(result.stderr)
return result.return_code
ZIM_NAME_PARSER = re.compile(r'(.+?)_(\d{4}-\d{2}).zim')
@ -566,10 +566,9 @@ async def restart_kiwix():
logger.info('Restarting Kiwix serve')
cmd = f'sudo /usr/bin/systemctl restart wrolpi-kiwix.service'
proc = await asyncio.create_subprocess_shell(cmd)
stdout, stderr = await proc.communicate()
logger.debug(f'systemctl returned {proc.returncode}')
if proc.returncode != 0 and stderr:
logger.debug(stderr)
return proc.returncode
cmd = ('sudo', '/usr/bin/systemctl', 'restart' 'wrolpi-kiwix.service')
result = await run_command(cmd)
logger.debug(f'systemctl returned {result.return_code}')
if result.return_code != 0 and result.stderr:
logger.debug(result.stderr)
return result.return_code

View file

@ -12,6 +12,7 @@ from wrolpi.common import DownloadFileInfo, get_wrolpi_config
from wrolpi.downloader import Download, import_downloads_config, save_downloads_config
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.test.common import skip_circleci
@pytest.mark.asyncio
@ -182,6 +183,7 @@ def test_zim_all_entries(test_session, test_zim):
assert len(entries) == 12
@skip_circleci
@pytest.mark.asyncio
async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock,
await_switches):