rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Creating a new Collection model for use in organizing FileGroups.

Browse source 
Replacing Channels and Domains with Collection.

Moving downloading relationships from Channel to Collection.

Created domains.yaml config to manage tagged domain collections.

Adding turn-key upgrade solution in the UI.
This commit is contained in:
Roland 2025-10-29 22:16:33 -06:00
parent 1f7b1c8744
commit 4fae4ae1b4
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
115 changed files with 13309 additions and 986 deletions
Download patch file Download diff file Expand all files Collapse all files

79
modules/archive/__init__.py
View file

@ -8,6 +8,7 @@ from sqlalchemy import not_
from sqlalchemy.orm import Session
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM
from wrolpi.collections import Collection
from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
slow_logger, get_title_from_html
from wrolpi.db import optional_session, get_db_session
@ -18,8 +19,8 @@ from wrolpi.vars import PYTEST, DOCKERIZED, DOWNLOAD_USER_AGENT
from . import lib
from .api import archive_bp # noqa
from .errors import InvalidArchive
from .lib import is_singlefile_file, request_archive, SINGLEFILE_HEADER
from .models import Archive, Domain
from .lib import is_singlefile_file, request_archive, SINGLEFILE_HEADER, get_url_from_singlefile
from .models import Archive
PRETTY_NAME = 'Archive'
@ -186,15 +187,61 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
if not archive:
# Create new Archive if it doesn't exist
archive = Archive(file_group_id=file_group_id, file_group=file_group)
# Set collection_id BEFORE adding to session to avoid autoflush constraint violation
from modules.archive.lib import get_or_create_domain_collection
if file_group.url:
# URL is already set on the FileGroup
collection = get_or_create_domain_collection(session, file_group.url)
archive.collection_id = collection.id if collection else None
else:
# No URL - try to extract from singlefile
try:
url = get_url_from_singlefile(singlefile_path.read_bytes())
file_group.url = url
collection = get_or_create_domain_collection(session, url)
archive.collection_id = collection.id if collection else None
except (RuntimeError, ValueError) as e:
# Could not extract URL from singlefile
if not PYTEST:
# In production, archives must have a URL/collection
raise InvalidArchive(f'Archive has no URL and could not extract from singlefile: {e}') from e
# In tests, allow archives without collections (factory will set it later)
logger.debug(f'Could not extract URL from singlefile (test mode): {e}')
archive.collection_id = None
session.add(archive)
archive.validate()
archive.flush()
# Validate that archive path matches domain directory (efficient O(1) check)
if archive.collection_id and archive.collection:
collection = archive.collection
if collection.kind == 'domain' and collection.directory and not collection.tag_id:
# Check if archive path is under the domain's directory
archive_path = pathlib.Path(file_group.primary_path)
from wrolpi.common import get_media_directory
media_dir = get_media_directory()
collection_abs_dir = media_dir / collection.directory
try:
# Check if archive is under the collection's directory
archive_path.relative_to(collection_abs_dir)
except ValueError:
# Archive is NOT under the domain's directory - clear it
logger.warning(
f'Archive {archive.id} at {archive_path} is not under domain directory {collection.directory}. '
f'Clearing domain directory (domain is not tagged).'
)
collection.directory = None
session.flush([collection])
file_group.title = file_group.a_text = title or archive.file_group.title
file_group.d_text = contents
file_group.data = {
'id': archive.id,
'domain': archive.domain.domain if archive.domain else None,
'domain': archive.domain,
'readability_json_path': archive.readability_json_path,
'readability_path': archive.readability_path,
'readability_txt_path': archive.readability_txt_path,
@ -230,26 +277,33 @@ async def archive_modeler():
processed = 0
for processed, (file_group, archive) in enumerate(results):
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
file_group.indexed = True
with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}',
logger__=logger):
if archive:
try:
archive_id = archive.id
archive.validate()
# Successfully validated, mark as indexed
file_group.indexed = True
except Exception:
logger.error(f'Unable to validate Archive {archive_id}')
# Don't mark as indexed - will retry later
if PYTEST:
raise
else:
try:
model_archive(file_group, session=session)
# Successfully modeled, mark as indexed
file_group.indexed = True
except InvalidArchive:
# It was not a real Archive. Many HTML files will not be an Archive.
file_group.indexed = False
invalid_archives.add(file_group.id)
except Exception as e:
# Some other error occurred during modeling - don't mark as indexed so we can retry
logger.error(f'Failed to model Archive for FileGroup {file_group.id}: {e}')
if PYTEST:
raise
session.commit()
@ -267,11 +321,14 @@ async def archive_modeler():
@limit_concurrent(1)
def archive_cleanup():
with get_db_session(commit=True) as session:
# Remove any Domains without any Archives.
domain_ids = [i[0] for i in session.execute('SELECT DISTINCT domain_id FROM archive') if i[0]]
for domain in session.query(Domain):
if domain.id not in domain_ids:
session.delete(domain)
# Remove any domain Collections without any Archives.
# Get all collection_ids that have archives
collection_ids = [i[0] for i in session.execute('SELECT DISTINCT collection_id FROM archive') if i[0]]
# Find domain collections that have no archives
for collection in session.query(Collection).filter_by(kind='domain'):
if collection.id not in collection_ids:
logger.info(f'Deleting empty domain collection: {collection.name}')
session.delete(collection)
def get_title(path):

10
modules/archive/api.py
View file

@ -9,10 +9,8 @@ from sqlalchemy.orm import Session
from wrolpi.api_utils import json_response, api_app
from wrolpi.common import logger, wrol_mode_check, api_param_limiter, TRACE_LEVEL
from wrolpi.db import get_db_session
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.files.lib import upsert_file
from wrolpi.schema import JSONErrorResponse
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from . import lib, schema
@ -48,14 +46,6 @@ async def delete_archive(_: Request, archive_ids: str):
return response.empty()
@archive_bp.get('/domains')
@openapi.summary('Get a list of all Domains and their Archive statistics')
@openapi.response(200, schema.GetDomainsResponse, "The list of domains")
async def get_domains(_: Request):
domains = lib.get_domains()
return json_response({'domains': domains, 'totals': {'domains': len(domains)}})
archive_limit_limiter = api_param_limiter(100)

29
modules/archive/conftest.py
View file

@ -8,7 +8,8 @@ import pytest
import pytz
from modules.archive.lib import archive_strftime
from modules.archive.models import Archive, Domain
from modules.archive.models import Archive
from wrolpi.collections import Collection
@pytest.fixture
@ -58,21 +59,27 @@ def archive_factory(test_session, archive_directory, make_files_structure, image
})
if domain:
domain = test_session.query(Domain).filter_by(domain=domain).one_or_none()
# Find or create domain collection
collection = test_session.query(Collection).filter_by(
name=domain,
kind='domain'
).one_or_none()
if not collection:
collection = Collection(
name=domain,
kind='domain',
directory=None, # Domain collections are unrestricted
)
test_session.add(collection)
test_session.flush([collection])
else:
collection = None
screenshot_path = None
if screenshot:
screenshot_path = domain_dir / f'{timestamp}_{title}.png'
screenshot_path.write_bytes(image_bytes_factory())
if not domain and domain_dir.name != 'NA':
domain = Domain(
domain=domain_dir.name,
directory=domain_dir,
)
test_session.add(domain)
test_session.flush([domain])
# Only add files that were created.
files = (readability_path, readability_json_path, readability_txt_path, screenshot_path, singlefile_path)
files = list(filter(None, files))
@ -82,7 +89,7 @@ def archive_factory(test_session, archive_directory, make_files_structure, image
archive.title = title
archive.file_group.download_datetime = archive.file_group.published_datetime = next(now)
archive.file_group.modification_datetime = next(now)
archive.domain = domain
archive.collection = collection
archive.validate()
for tag_name in tag_names:

535
modules/archive/lib.py
View file

@ -7,7 +7,7 @@ import pathlib
import re
import shlex
import tempfile
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime
from json import JSONDecodeError
from typing import Optional, Tuple, List, Union
@ -17,22 +17,190 @@ import pytz
from sqlalchemy import asc
from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from modules.archive.models import Archive
from wrolpi import dates
from wrolpi.cmd import READABILITY_BIN, run_command
from wrolpi.collections import Collection
from wrolpi.common import get_media_directory, logger, extract_domain, escape_file_name, aiohttp_post, \
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config, html_screenshot
format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, get_wrolpi_config, html_screenshot, \
ConfigFile
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
from wrolpi.events import Events
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.tags import tag_append_sub_select_where
from wrolpi.vars import PYTEST, DOCKERIZED
logger = logger.getChild(__name__)
__all__ = ['DomainsConfig', 'domains_config', 'get_domains_config', 'save_domains_config', 'import_domains_config']
ARCHIVE_SERVICE = 'http://archive:8080'
@dataclass
class DomainsConfigValidator:
"""Validator for domains config file."""
version: int = 0
collections: List[dict] = field(default_factory=list)
class DomainsConfig(ConfigFile):
"""
Config file for Domain Collections.
This is a specialized config that manages domain collections
(Collections with kind='domain'). It maintains a domains.yaml file for
backward compatibility and user convenience.
Format:
collections:
- name: "example.com"
kind: "domain"
description: "Archives from example.com"
- name: "wikipedia.org"
kind: "domain"
Note: Domain collections can have optional directories for tagging support.
Items are managed dynamically when Archives are indexed.
"""
file_name = 'domains.yaml'
validator = DomainsConfigValidator
default_config = dict(
version=0,
collections=[],
)
# Use wider width to accommodate longer paths
width = 120
def __getitem__(self, item):
return self._config[item]
def __setitem__(self, key, value):
self._config[key] = value
@property
def collections(self) -> List[dict]:
"""Get list of collection configs."""
return self._config.get('collections', [])
def import_config(self, file: pathlib.Path = None, send_events=False):
"""Import domain collections from config file into database."""
ConfigFile.import_config(self, file, send_events)
file_str = str(self.get_relative_file())
collections_data = self._config.get('collections', [])
if not collections_data:
logger.info(f'No domain collections to import from {file_str}')
self.successful_import = True
return
logger.info(f'Importing {len(collections_data)} domain collections from {file_str}')
try:
with get_db_session(commit=True) as session:
# Track imported domain names
imported_domains = set()
# Import each domain collection, forcing kind='domain'
for idx, collection_data in enumerate(collections_data):
try:
name = collection_data.get('name')
if not name:
logger.error(f'Domain collection at index {idx} has no name, skipping')
continue
# Ensure kind is 'domain'
collection_data = collection_data.copy()
collection_data['kind'] = 'domain'
# Warn if tag_name provided without directory
tag_name = collection_data.get('tag_name')
directory = collection_data.get('directory')
if tag_name and not directory:
logger.warning(
f"Domain collection '{name}' has tag_name '{tag_name}' "
f"but no directory - tags require a directory. Tag will be ignored."
)
collection_data.pop('tag_name', None)
# Use Collection.from_config to create/update
collection = Collection.from_config(collection_data, session)
imported_domains.add(collection.name)
except Exception as e:
logger.error(f'Failed to import domain collection at index {idx}', exc_info=e)
continue
# Delete domain collections that are no longer in config
all_domain_collections = session.query(Collection).filter_by(kind='domain').all()
for collection in all_domain_collections:
if collection.name not in imported_domains:
logger.info(f'Deleting domain collection {repr(collection.name)} (no longer in config)')
session.delete(collection)
logger.info(f'Successfully imported {len(imported_domains)} domain collections from {file_str}')
self.successful_import = True
except Exception as e:
self.successful_import = False
message = f'Failed to import {file_str} config!'
logger.error(message, exc_info=e)
if send_events:
Events.send_config_import_failed(message)
raise
def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False):
"""Dump all domain collections from database to config file."""
logger.info('Dumping domain collections to config')
with get_db_session() as session:
# Get only domain collections, ordered by name
collections = session.query(Collection).filter_by(kind='domain').order_by(Collection.name).all()
# Use to_config to export each collection
collections_data = [collection.to_config() for collection in collections]
self._config['collections'] = collections_data
logger.info(f'Dumping {len(collections_data)} domain collections to config')
self.save(file, send_events, overwrite)
# Global instance
domains_config = DomainsConfig()
def get_domains_config() -> DomainsConfig:
"""Get the global domains config instance."""
return domains_config
# Switch handler for saving domains config
@register_switch_handler('save_domains_config')
def save_domains_config():
"""Save the domains config when the switch is activated."""
domains_config.background_dump.activate_switch()
# Explicit type for activate_switch helper
save_domains_config: ActivateSwitchMethod
def import_domains_config():
"""Import domain collections from config file into database."""
logger.info('Importing domains config')
domains_config.import_config()
# Link downloads to domain collections after import
with get_db_session() as session:
link_domain_and_downloads(session)
logger.info('Importing domains config completed')
@dataclass
class ArchiveFiles:
"""Every Archive will have some of these files."""
@ -74,6 +242,25 @@ def get_domain_directory(url: str) -> pathlib.Path:
return directory
def get_archive_destination(domain_collection: 'Collection') -> pathlib.Path:
"""
Get the destination directory for archives in a domain collection.
Args:
domain_collection: The domain Collection (kind='domain')
Returns:
Path where archives should be placed. If collection has a directory,
returns that directory. Otherwise returns the default archives/<domain>/ path.
"""
if domain_collection.directory:
# Restricted collection - use collection directory
return get_media_directory() / domain_collection.directory
else:
# Unrestricted collection - use default archive/<domain>/ path
return get_archive_directory() / domain_collection.name
# File names include domain and datetime.
MAXIMUM_ARCHIVE_FILE_CHARACTER_LENGTH = 200
@ -248,24 +435,223 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
archive = Archive.from_paths(session, *paths)
archive.file_group.download_datetime = now()
archive.url = url
archive.domain = get_or_create_domain(session, url)
archive.collection = get_or_create_domain_collection(session, url)
archive.flush()
archive.domain.flush()
return archive
def get_or_create_domain(session: Session, url) -> Domain:
def detect_domain_directory(collection: Collection, session: Session) -> Optional[pathlib.Path]:
"""
Get/create the Domain for this archive.
Detect if all archives for a domain collection share a common directory.
Args:
collection: The domain collection to analyze
session: Database session
Returns:
Path (relative to media directory) if all archives share a common directory within archive media directory.
None if archives are scattered across different directories or if collection has no archives.
"""
domain_ = extract_domain(url)
domain = session.query(Domain).filter_by(domain=domain_).one_or_none()
if not domain:
domain = Domain(domain=domain_, directory=str(get_domain_directory(url)))
session.add(domain)
if collection.kind != 'domain':
return None
# Query all archives for this domain collection
archives = session.query(Archive).filter_by(collection_id=collection.id).all()
if not archives:
# No archives yet, can't determine directory
return None
# Get all archive file paths
paths = []
for archive in archives:
if archive.file_group and archive.file_group.primary_path:
paths.append(pathlib.Path(archive.file_group.primary_path))
if not paths:
return None
# Find common ancestor directory
# Start with the first path's parent directory
common_dir = paths[0].parent
# Check if all other paths are under this directory
for path in paths[1:]:
try:
# Check if path is relative to common_dir
path.relative_to(common_dir)
except ValueError:
# Path is not under common_dir, find the common ancestor
# Walk up until we find a common parent
while common_dir != common_dir.parent: # Stop at root
try:
path.relative_to(common_dir)
break # Found common ancestor
except ValueError:
common_dir = common_dir.parent
# Check if common directory is within the media directory
media_dir = get_media_directory()
try:
relative_path = common_dir.relative_to(media_dir)
except ValueError:
# Common directory is outside media directory
return None
# Check if it's within the archive directory structure
archive_base = get_archive_directory()
try:
archive_base.relative_to(media_dir) # Verify archive_base is under media_dir
common_dir.relative_to(archive_base.parent) # Verify common_dir is under archive structure
except ValueError:
# Not in the archive directory structure
return None
logger.debug(f'Detected directory for domain {collection.name}: {relative_path}')
return relative_path
def update_domain_directories(session: Session = None) -> int:
"""
One-time update to detect and set directories for existing domain collections.
This should be run once to fix domain collections that were created without directories
but have all their archives in a common location.
Args:
session: Database session
Returns:
Number of domain collections updated
"""
if session is None:
with get_db_session(commit=True) as session:
return update_domain_directories(session)
# Find all domain collections without directories
collections = session.query(Collection).filter_by(kind='domain', directory=None).all()
updated_count = 0
for collection in collections:
detected_dir = detect_domain_directory(collection, session)
if detected_dir:
collection.directory = detected_dir
session.flush([collection])
updated_count += 1
logger.info(f'Updated domain {collection.name} with directory: {detected_dir}')
session.commit()
logger.info(f'Updated {updated_count} domain collection(s) with auto-detected directories')
return updated_count
def get_or_create_domain_collection(session: Session, url, directory: pathlib.Path = None) -> Collection:
"""
Get or create the domain Collection for this archive.
Args:
session: Database session
url: URL of the archive
directory: Optional directory to restrict this domain collection.
If None, creates unrestricted collection (default).
If provided, archives will be placed in this directory and collection can be tagged.
Returns:
Collection with kind='domain' for the domain extracted from the URL
"""
domain_name = extract_domain(url)
# Try to find existing domain collection
collection = session.query(Collection).filter_by(
name=domain_name,
kind='domain'
).one_or_none()
if not collection:
# Create new domain collection
collection = Collection(
name=domain_name,
kind='domain',
directory=directory, # Can be None (unrestricted) or a Path (restricted)
)
session.add(collection)
session.flush()
return domain
# Trigger domain config save for new domain
save_domains_config.activate_switch()
if directory:
logger.info(f'Created domain collection with directory: {domain_name} -> {directory}')
else:
logger.info(f'Created unrestricted domain collection: {domain_name}')
# Auto-detect directory if not explicitly set and collection doesn't have one
if not directory and not collection.directory:
detected_dir = detect_domain_directory(collection, session)
if detected_dir:
collection.directory = detected_dir
session.flush()
# Trigger domain config save when directory is auto-detected
save_domains_config.activate_switch()
logger.info(f'Auto-detected directory for domain {domain_name}: {detected_dir}')
return collection
def link_domain_and_downloads(session: Session):
"""Associate any Download related to a Domain Collection.
Downloads are linked to Collections (via collection_id).
This function finds Domain Collections and links their Downloads.
Matching criteria:
- Recurring downloads whose destination is within the domain collection's directory (including subdirectories)
- RSS downloads with sub_downloader='archive' (matched by URL domain)
"""
from wrolpi.downloader import Download
from wrolpi.collections.models import Collection
# Only Downloads with a frequency can be a Collection Download.
downloads = list(session.query(Download).filter(Download.frequency.isnot(None)).all())
# Get domain collections that have a directory
domain_collections = session.query(Collection).filter(
Collection.kind == 'domain',
Collection.directory.isnot(None)
).all()
need_commit = False
# Match downloads by destination directory (including subdirectories)
downloads_with_destination = [d for d in downloads if (d.settings or {}).get('destination')]
for collection in domain_collections:
# Ensure directory ends with / for proper prefix matching
directory = str(collection.directory)
directory_prefix = directory if directory.endswith('/') else directory + '/'
for download in downloads_with_destination:
dest = download.settings['destination']
# Match if destination equals directory or is a subdirectory
if not download.collection_id and (dest == directory or dest.startswith(directory_prefix)):
download.collection_id = collection.id
need_commit = True
# Match RSS downloads with archive sub_downloader by URL domain
rss_archive_downloads = [
d for d in downloads
if d.downloader == 'rss' and d.sub_downloader == 'archive' and not d.collection_id
]
for download in rss_archive_downloads:
# Extract domain from the RSS URL and find matching domain collection
domain_name = extract_domain(download.url)
collection = session.query(Collection).filter_by(
name=domain_name,
kind='domain'
).one_or_none()
if collection:
download.collection_id = collection.id
need_commit = True
if need_commit:
session.commit()
SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n')
@ -463,18 +849,86 @@ def is_singlefile_file(path: pathlib.Path) -> bool:
def get_domains():
"""
Get all domain collections with their archive statistics.
This is a thin wrapper around Collection queries that adds archive-specific statistics.
Returns a list of dicts with collection id, domain name, url_count, and total size.
"""
from sqlalchemy import func, BigInteger
from wrolpi.files.models import FileGroup
with get_db_session() as session:
# Query all domain collections with archive statistics in a single query
# This uses ORM for better maintainability while keeping performance
query = (
session.query(
Collection.id,
Collection.name.label('domain'),
func.count(Archive.id).label('url_count'),
func.sum(FileGroup.size).cast(BigInteger).label('size')
)
.outerjoin(Archive, Collection.id == Archive.collection_id)
.outerjoin(FileGroup, FileGroup.id == Archive.file_group_id)
.filter(Collection.kind == 'domain')
.group_by(Collection.id, Collection.name)
.order_by(Collection.name)
)
domains = [
{
'id': row.id,
'domain': row.domain,
'url_count': row.url_count or 0,
'size': row.size or 0,
}
for row in query.all()
]
return domains
def get_domain(domain_id: int) -> dict:
"""
Get a single domain collection by ID with its archive statistics.
Returns a dict with collection details including id, domain name, url_count, size,
tag_name, directory (relative to media directory), and description.
Raises UnknownArchive if domain not found.
"""
# Get the collection using find_by_id
try:
collection = Collection.find_by_id(domain_id)
except Exception:
# Collection.find_by_id raises UnknownCollection, but we want UnknownArchive
raise UnknownArchive(f"Domain collection with ID {domain_id} not found")
if collection.kind != 'domain':
raise UnknownArchive(f"Collection {domain_id} is not a domain")
# Get base domain data from __json__()
domain_data = collection.__json__()
# Get archive statistics for this domain
with get_db_curs() as curs:
stmt = '''
SELECT domains.domain AS domain, COUNT(a.id) AS url_count, SUM(fg.size)::BIGINT AS size
FROM domains
LEFT JOIN archive a on domains.id = a.domain_id
SELECT COUNT(a.id) AS url_count, SUM(fg.size)::BIGINT AS size
FROM archive a
LEFT JOIN file_group fg on fg.id = a.file_group_id
GROUP BY domains.domain
ORDER BY domains.domain \
WHERE a.collection_id = %(domain_id)s
'''
curs.execute(stmt)
domains = [dict(i) for i in curs.fetchall()]
return domains
curs.execute(stmt, {'domain_id': domain_id})
stats = dict(curs.fetchone())
# Enhance with archive stats
domain_data['url_count'] = stats['url_count'] or 0
domain_data['size'] = stats['size'] or 0
# Use 'domain' key for domain collections instead of 'name'
domain_data['domain'] = domain_data.pop('name')
return domain_data
ARCHIVE_ORDERS = {
@ -556,7 +1010,9 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
if domain:
params['domain'] = domain
wheres.append('a.domain_id = (select id from domains where domains.domain = %(domain)s)')
# Use LIMIT 1 to handle potential duplicate domain collections with the same name
wheres.append(
"a.collection_id = (select id from collection where collection.name = %(domain)s and collection.kind = 'domain' LIMIT 1)")
select_columns = f", {select_columns}" if select_columns else ""
wheres = '\n AND '.join(wheres)
@ -582,13 +1038,38 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
@optional_session
async def search_domains_by_name(name: str, limit: int = 5, session: Session = None) -> List[Domain]:
domains = session.query(Domain) \
.filter(Domain.domain.ilike(f'%{name}%')) \
.order_by(asc(Domain.domain)) \
async def search_domains_by_name(name: str, limit: int = 5, session: Session = None) -> List[dict]:
"""
Search for domain collections by name.
Args:
name: Search string to match against collection names
limit: Maximum number of results to return
session: Database session
Returns:
List of domain dicts matching the search (in old Domain format for backward compatibility)
"""
collections = session.query(Collection) \
.filter(Collection.kind == 'domain') \
.filter(Collection.name.ilike(f'%{name}%')) \
.order_by(asc(Collection.name)) \
.limit(limit) \
.all()
return domains
# Convert to old Domain format for backward compatibility
from wrolpi.common import get_relative_to_media_directory
archive_dir = get_archive_directory()
return [
{
'id': c.id,
'domain': c.name,
# Domain collections can have explicit directories or use default archive path
'directory': get_relative_to_media_directory(c.directory) if c.directory else str(
(archive_dir / c.name).relative_to(get_media_directory())),
}
for c in collections
]
async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):

95
modules/archive/models.py
View file

@ -4,24 +4,23 @@ import re
from typing import Iterable, List, Optional
import pytz
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
from sqlalchemy.orm import relationship, Session, validates
from sqlalchemy.orm.collections import InstrumentedList
from sqlalchemy import Column, Integer, ForeignKey, BigInteger
from sqlalchemy.orm import relationship, Session
from wrolpi import dates
from wrolpi.collections import Collection
from wrolpi.common import ModelHelper, Base, logger, get_title_from_html, get_wrolpi_config, get_media_directory
from wrolpi.dates import now
from wrolpi.db import optional_session
from wrolpi.errors import UnknownArchive
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
from wrolpi.tags import TagFile
from wrolpi.vars import PYTEST
from .errors import InvalidArchive
logger = logger.getChild(__name__)
__all__ = ['Archive', 'Domain']
__all__ = ['Archive']
MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
@ -31,17 +30,33 @@ class Archive(Base, ModelHelper):
__tablename__ = 'archive'
id = Column(Integer, primary_key=True)
domain_id = Column(Integer, ForeignKey('domains.id'))
domain = relationship('Domain', primaryjoin='Archive.domain_id==Domain.id')
collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'))
collection: Collection = relationship('Collection', primaryjoin='Archive.collection_id==Collection.id')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
file_group: FileGroup = relationship('FileGroup')
def __repr__(self):
if self.domain:
domain_name = self.domain if self.collection else None
if domain_name:
return f'<Archive id={self.id} url={self.file_group.url} singlefile={repr(str(self.singlefile_path))} ' \
f'domain={self.domain.domain}>'
f'domain={domain_name}>'
return f'<Archive id={self.id} url={self.file_group.url} singlefile={repr(str(self.singlefile_path))}>'
@property
def domain(self) -> str | None:
"""
Get the domain name for this Archive.
This property provides backward compatibility by returning the Collection's name
(which is the domain string for domain collections).
Returns:
The domain string (e.g., "example.com") or None if no collection
"""
if self.collection and self.collection.kind == 'domain':
return self.collection.name
return None
@staticmethod
@optional_session
def get_by_id(id_: int, session: Session = None) -> Optional['Archive']:
@ -116,14 +131,14 @@ class Archive(Base, ModelHelper):
self.file_group.delete()
session = Session.object_session(self)
collection = self.collection
session.delete(self)
if self.domain:
# Delete a domain if it has no Archives.
try:
next(i.id for i in self.domain.archives)
except StopIteration:
self.domain.delete()
if collection and collection.kind == 'domain':
# Delete a domain collection if it has no Archives.
remaining_archives = session.query(Archive).filter_by(collection_id=collection.id).count()
if remaining_archives == 0:
session.delete(collection)
@property
def history(self) -> Iterable[Base]:
@ -299,16 +314,16 @@ class Archive(Base, ModelHelper):
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
def apply_domain(self):
"""Get the domain from the URL."""
from modules.archive.lib import get_or_create_domain
domain = None
"""Get the domain collection from the URL."""
from modules.archive.lib import get_or_create_domain_collection
collection = None
if self.file_group.url:
session = Session.object_session(self)
if not session:
raise ValueError('No session found!')
domain = get_or_create_domain(session, self.file_group.url)
# Clear domain if the URL is missing.
self.domain_id = domain.id if domain else None
collection = get_or_create_domain_collection(session, self.file_group.url)
# Clear collection if the URL is missing.
self.collection_id = collection.id if collection else None
def apply_singlefile_title(self):
"""Get the title from the Singlefile, if it's missing."""
@ -363,39 +378,23 @@ class Archive(Base, ModelHelper):
"""The location where this Archive can be viewed in the UI."""
return f'/archive/{self.id}'
class Domain(Base, ModelHelper):
__tablename__ = 'domains' # plural to avoid conflict
id = Column(Integer, primary_key=True)
domain = Column(String, nullable=False)
directory = Column(MediaPathType)
archives: InstrumentedList = relationship('Archive', primaryjoin='Archive.domain_id==Domain.id')
def __repr__(self):
return f'<Domain id={self.id} domain={self.domain} directory={self.directory}>'
def delete(self):
session = Session.object_session(self)
session.execute('DELETE FROM archive WHERE domain_id=:id', dict(id=self.id))
session.query(Domain).filter_by(id=self.id).delete()
@validates('domain')
def validate_domain(self, key, value: str):
if not isinstance(value, str):
raise ValueError('Domain must be a string')
if len(value.split('.')) < 2:
raise ValueError(f'Domain must contain at least one "." domain={repr(value)}')
return value
@property
def download_directory(self) -> pathlib.Path:
"""
Get the download directory for this Archive based on its domain and date.
This uses the configured archive_destination template with variables:
- domain: the domain name from the Collection
- year, month, day: current date
Returns:
Absolute path where new archives for this domain should be downloaded
"""
archive_destination = get_wrolpi_config().archive_destination
now_ = now()
variables = dict(
domain=self.domain,
domain=self.domain or 'unknown',
year=now_.year,
month=now_.month,
day=now_.day,

12
modules/archive/schema.py
View file

@ -15,18 +15,6 @@ class ArchiveDict:
title: str
@dataclass
class DomainDict:
domain: str
url_count: int
size: int
@dataclass
class GetDomainsResponse:
domains: List[DomainDict]
@dataclass
class ArchiveSearchRequest:
search_str: Optional[str] = None

127
modules/archive/test/test_api.py
View file

@ -5,14 +5,12 @@ from unittest.mock import patch
import pytest
from modules.archive import lib, Archive
from wrolpi.files.models import FileGroup
from wrolpi.common import get_relative_to_media_directory
from wrolpi.test.common import skip_circleci, skip_macos
from wrolpi.common import get_relative_to_media_directory
def check_results(test_client, data, ids):
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
async def check_results(async_client, data, ids):
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
if not response.status_code == HTTPStatus.OK:
raise AssertionError(str(response.json))
@ -23,25 +21,27 @@ def check_results(test_client, data, ids):
f'{response.json["file_groups"][0]["id"]}..{response.json["file_groups"][-1]["id"]}'
def test_archives_search_order(test_session, archive_directory, archive_factory, test_client):
@pytest.mark.asyncio
async def test_archives_search_order(test_session, archive_directory, archive_factory, async_client):
"""Search using all orders."""
archive_factory('example.com', 'https://example.com/one', 'my archive', 'foo bar qux')
for order_by in lib.ARCHIVE_ORDERS:
data = {'search_str': 'foo', 'order_by': order_by}
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
if not response.status_code == HTTPStatus.OK:
raise AssertionError(str(response.json))
data = {'search_str': None, 'order_by': order_by}
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
if not response.status_code == HTTPStatus.OK:
raise AssertionError(str(response.json))
def test_archives_search(test_session, archive_directory, archive_factory, test_client):
@pytest.mark.asyncio
async def test_archives_search(test_session, archive_directory, archive_factory, async_client):
"""Archives can be searched by their title and their contents."""
# Search with no archives.
check_results(test_client, {'search_str': 'foo'}, [])
await check_results(async_client, {'search_str': 'foo'}, [])
archive_factory('example.com', 'https://example.com/one', 'my archive', 'foo bar qux')
archive_factory('example.com', 'https://example.com/one', 'other archive', 'foo baz qux qux')
@ -51,38 +51,38 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_
# 1 and 2 contain "foo".
data = {'search_str': 'foo'}
check_results(test_client, data, [2, 1])
await check_results(async_client, data, [2, 1])
# 2 and 3 contain "baz".
data = {'search_str': 'baz'}
check_results(test_client, data, [3, 2])
await check_results(async_client, data, [3, 2])
# 1 contains "bar".
data = {'search_str': 'bar'}
check_results(test_client, data, [1, ])
await check_results(async_client, data, [1, ])
# No archives contain "huzzah"
data = {'search_str': 'huzzah'}
check_results(test_client, data, [])
await check_results(async_client, data, [])
# Only 3 contains "baz" and is in domain "example.org"
data = {'search_str': 'baz', 'domain': 'example.org'}
check_results(test_client, data, [3, ])
await check_results(async_client, data, [3, ])
# 1's title contains "my", this is ignored by Postgres.
data = {'search_str': 'my'}
check_results(test_client, data, [])
await check_results(async_client, data, [])
# 3's title contains "third".
data = {'search_str': 'third'}
check_results(test_client, data, [3, ])
await check_results(async_client, data, [3, ])
# All contents contain "qux", but they contain different amounts. They are ordered by the amount.
data = {'search_str': 'qux'}
check_results(test_client, data, [3, 2, 1])
await check_results(async_client, data, [3, 2, 1])
data = {'search_str': 'qux', 'order_by': 'bad order_by'}
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
@ -106,7 +106,8 @@ async def test_search_archive_tags(test_session, async_client, archive_factory,
assert response.json['totals']['file_groups'] == 0
def test_archives_search_headline(test_session, archive_directory, archive_factory, test_client):
@pytest.mark.asyncio
async def test_archives_search_headline(test_session, archive_directory, archive_factory, async_client):
"""Headlines can be requested."""
archive_factory('example.com', 'https://example.com/one', 'my archive', 'foo bar qux')
archive_factory('example.com', 'https://example.com/one', 'other archive', 'foo baz qux qux')
@ -115,7 +116,7 @@ def test_archives_search_headline(test_session, archive_directory, archive_facto
test_session.commit()
content = dict(search_str='foo')
request, response = test_client.post('/api/archive/search', content=json.dumps(content))
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
# Headlines are only fetched if requested.
@ -123,7 +124,7 @@ def test_archives_search_headline(test_session, archive_directory, archive_facto
assert response.json['file_groups'][1]['d_headline'] is None
content = dict(search_str='foo', headline=True)
request, response = test_client.post('/api/archive/search', content=json.dumps(content))
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
# Postgresql uses <b>...</b> to highlight matching words.
@ -131,25 +132,27 @@ def test_archives_search_headline(test_session, archive_directory, archive_facto
assert response.json['file_groups'][1]['d_headline'] == '<b>foo</b> bar qux'
def test_search_offset(test_session, archive_factory, test_client):
@pytest.mark.asyncio
async def test_search_offset(test_session, archive_factory, async_client):
"""Archive search can be offset."""
for i in range(500):
archive_factory('example.com', f'https://example.com/{i}', contents='foo bar')
test_session.commit()
data = {'search_str': None, 'offset': 0}
check_results(test_client, data, list(range(500, 480, -1)))
await check_results(async_client, data, list(range(500, 480, -1)))
data = {'search_str': None, 'offset': 20}
check_results(test_client, data, list(range(480, 460, -1)))
await check_results(async_client, data, list(range(480, 460, -1)))
data = {'search_str': None, 'offset': 100}
check_results(test_client, data, list(range(400, 380, -1)))
await check_results(async_client, data, list(range(400, 380, -1)))
data = {'search_str': None, 'offset': 200}
check_results(test_client, data, list(range(300, 280, -1)))
await check_results(async_client, data, list(range(300, 280, -1)))
data = {'search_str': None, 'offset': 500}
check_results(test_client, data, [])
await check_results(async_client, data, [])
def test_archives_search_no_query(test_session, archive_factory, test_client):
@pytest.mark.asyncio
async def test_archives_search_no_query(test_session, archive_factory, async_client):
"""Archive Search API endpoint does not require data in the body."""
# Add 100 random archives.
for _ in range(100):
@ -157,83 +160,26 @@ def test_archives_search_no_query(test_session, archive_factory, test_client):
test_session.commit()
# All archives are returned when no `search_str` is passed.
request, response = test_client.post('/api/archive/search', content='{}')
request, response = await async_client.post('/api/archive/search', content='{}')
assert response.status_code == HTTPStatus.OK, response.json
assert [i['id'] for i in response.json['file_groups']] == list(range(100, 80, -1))
assert response.json['totals']['file_groups'] == 100
# All archives are from "example.com".
data = dict(domain='example.com')
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK, response.json
assert [i['id'] for i in response.json['file_groups']] == list(range(100, 80, -1))
assert response.json['totals']['file_groups'] == 100
# No archives are from "example.org".
data = dict(domain='example.org')
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK, response.json
assert [i['id'] for i in response.json['file_groups']] == []
assert response.json['totals']['file_groups'] == 0
def test_archive_and_domain_crud(test_session, test_client, archive_factory):
"""Getting an Archive returns it's File. Testing deleting Archives."""
# Can get empty results.
request, response = test_client.get(f'/api/archive/1')
assert response.status_code == HTTPStatus.NOT_FOUND
request, response = test_client.get(f'/api/archive/domains')
assert response.status_code == HTTPStatus.OK
assert response.json['domains'] == []
archive1 = archive_factory(domain='example.com', url='https://example.com/1')
archive2 = archive_factory(domain='example.com', url='https://example.com/1')
test_session.commit()
# Archive1 has Archive2 as history.
request, response = test_client.get(f'/api/archive/{archive1.id}')
assert response.status_code == HTTPStatus.OK
assert response.json['file_group']['id'] == archive1.id
assert response.json['history'][0]['id'] == archive2.id
# Archive2 has Archive1 as history.
request, response = test_client.get(f'/api/archive/{archive2.id}')
assert response.status_code == HTTPStatus.OK
assert archive2.id == response.json['file_group']['id']
assert response.json['history'][0]['id'] == archive1.id
# Only one domain.
request, response = test_client.get(f'/api/archive/domains')
assert response.status_code == HTTPStatus.OK
assert response.json['domains'][0]['domain'] == 'example.com'
assert response.json['domains'][0]['size'] == 254
# Deleting works.
request, response = test_client.delete(f'/api/archive/{archive1.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
# Trying to delete again returns NOT_FOUND.
request, response = test_client.delete(f'/api/archive/{archive1.id}')
assert response.status_code == HTTPStatus.NOT_FOUND
# Can't get deleted Archive.
request, response = test_client.get(f'/api/archive/{archive1.id}')
assert response.status_code == HTTPStatus.NOT_FOUND
# Archive2 no longer has history.
request, response = test_client.get(f'/api/archive/{archive2.id}')
assert response.status_code == HTTPStatus.OK
assert response.json['file_group']['id'] == archive2.id
assert response.json['history'] == []
# No Archives, no Domains.
request, response = test_client.delete(f'/api/archive/{archive2.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = test_client.get(f'/api/archive/domains')
assert response.status_code == HTTPStatus.OK
assert response.json['domains'] == []
@skip_macos
@skip_circleci
@pytest.mark.asyncio
@ -356,7 +302,8 @@ async def test_archive_upload_file_tracking(test_session, async_client, archive_
# Assert screenshot properties exist
assert archive.screenshot_path is not None, 'Archive.screenshot_path should exist'
assert archive.screenshot_file is not None, 'Archive.screenshot_file should exist'
assert archive.file_group.data.get('screenshot_path') == archive.screenshot_path, 'Archive screenshot should be in FileGroup.data'
assert archive.file_group.data.get(
'screenshot_path') == archive.screenshot_path, 'Archive screenshot should be in FileGroup.data'
# Assert image is in FileGroup.files
file_paths = [f['path'] for f in archive.file_group.files]

139
modules/archive/test/test_collection_id_bug.py Normal file
View file

@ -0,0 +1,139 @@
"""Test for collection_id bug when modeling archives from discovered files.
This test reproduces the bug where Archives created by model_archive()
don't have collection_id set, causing a NOT NULL constraint violation.
The bug happens because:
1. The archive_factory sets collection_id explicitly (masking the bug)
2. But model_archive() (used in production) doesn't set collection_id
3. This causes NULL constraint violations when indexing real files
"""
import pathlib
import pytest
from modules.archive import model_archive
from modules.archive.lib import archive_strftime
from wrolpi.collections import Collection
@pytest.mark.asyncio
async def test_model_archive_sets_collection_id(async_client, test_session, test_directory, make_files_structure):
"""
Test that model_archive() creates Archives with collection_id set.
This test uses model_archive() (the production code path) instead of
archive_factory, so it will detect the collection_id bug.
"""
import datetime
import pytz
# Create archive files with a URL in the singlefile
domain = 'example.com'
url = f'https://{domain}/test-page'
archive_dir = test_directory / 'archives' / domain
archive_dir.mkdir(parents=True)
timestamp = archive_strftime(datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC))
title = 'Test Page'
# Create a minimal singlefile with URL embedded (note trailing space after SingleFile)
singlefile_content = f'''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: {url}
saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">
<meta name="generator" content="SingleFile">
<title>{title}</title>
</head>
<body>
<h1>{title}</h1>
<p>Test content</p>
</body>
</html>'''
files = make_files_structure({
str(archive_dir / f'{timestamp}_{title}.html'): singlefile_content.strip(),
str(archive_dir / f'{timestamp}_{title}.readability.json'): '{"title": "' + title + '"}',
})
from wrolpi.files.models import FileGroup
# Create FileGroup from the files (simulating what refresh does)
file_paths = [pathlib.Path(f) for f in files]
file_group = FileGroup.from_paths(test_session, *file_paths)
# Model the archive using the production code path
# This should create an Archive with collection_id set
archive = model_archive(file_group, session=test_session)
# The bug: collection_id is None, causing NOT NULL constraint violation
assert archive is not None, "Archive should be created"
assert archive.collection_id is not None, \
"Archive.collection_id should be set (this is the bug!)"
# Verify the collection was created
collection = test_session.query(Collection).filter_by(
name=domain,
kind='domain'
).one_or_none()
assert collection is not None, "Domain collection should be created"
assert archive.collection_id == collection.id, \
"Archive should be linked to the domain collection"
@pytest.mark.asyncio
async def test_model_archive_extracts_url_from_singlefile(async_client, test_session, test_directory, make_files_structure):
"""
Test that model_archive() can extract URL from singlefile when file_group.url is None.
This is the second part of the fix - if the URL isn't in the file_group yet,
we need to extract it from the singlefile content.
"""
import datetime
import pytz
# Create archive files
domain = 'test.org'
url = f'https://{domain}/article'
archive_dir = test_directory / 'archives' / domain
archive_dir.mkdir(parents=True)
timestamp = archive_strftime(datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC))
# Create singlefile with URL in saved-from comment
singlefile_content = f'''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: {url}
saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">
<meta name="generator" content="SingleFile">
<title>Test Article</title>
</head>
<body><p>Content</p></body>
</html>'''
files = make_files_structure({
str(archive_dir / f'{timestamp}_Test.html'): singlefile_content.strip(),
})
from wrolpi.files.models import FileGroup
# Create FileGroup from the files (simulating what refresh does)
file_paths = [pathlib.Path(f) for f in files]
file_group = FileGroup.from_paths(test_session, *file_paths)
# Model the archive - it should extract the URL from the singlefile
archive = model_archive(file_group, session=test_session)
assert archive is not None
assert archive.file_group.url == url, \
"URL should be extracted from singlefile content"
assert archive.collection_id is not None, \
"collection_id should be set after URL extraction"
# Verify collection was created with correct domain
collection = test_session.query(Collection).get(archive.collection_id)
assert collection.name == domain
assert collection.kind == 'domain'

313
modules/archive/test/test_domain_tagging.py Normal file
View file

@ -0,0 +1,313 @@
"""Tests for domain collection tagging functionality."""
import pytest
from wrolpi.collections import Collection
@pytest.mark.asyncio
async def test_create_domain_collection_with_directory(test_session, test_directory):
"""Domain collection can be created with a directory."""
domain_dir = test_directory / 'archives' / 'example.com'
domain_dir.mkdir(parents=True)
collection = Collection(
name='example.com',
kind='domain',
directory=domain_dir
)
test_session.add(collection)
test_session.commit()
assert collection.directory == domain_dir
assert collection.can_be_tagged is True
@pytest.mark.asyncio
async def test_create_unrestricted_domain_collection(test_session):
"""Unrestricted domain collection (no directory) cannot be tagged."""
collection = Collection(
name='example.com',
kind='domain',
directory=None # Unrestricted
)
test_session.add(collection)
test_session.commit()
assert collection.directory is None
assert collection.can_be_tagged is False
@pytest.mark.asyncio
async def test_tag_domain_collection_with_directory(async_client, test_session, test_directory, tag_factory):
"""Domain collection with directory can be tagged."""
domain_dir = test_directory / 'archives' / 'example.com'
domain_dir.mkdir(parents=True, exist_ok=True)
collection = Collection(
name='example.com',
kind='domain',
directory=domain_dir
)
test_session.add(collection)
tag = await tag_factory(name='News')
test_session.commit()
# Tag the collection
collection.set_tag('News')
test_session.commit()
assert collection.tag is not None
assert collection.tag.name == 'News'
assert collection.tag_id == tag.id
@pytest.mark.asyncio
async def test_cannot_tag_unrestricted_domain_collection(async_client, test_session, tag_factory):
"""Domain collection without directory cannot be tagged."""
collection = Collection(
name='example.com',
kind='domain',
directory=None # Unrestricted
)
test_session.add(collection)
tag = await tag_factory(name='News')
test_session.commit()
assert collection.can_be_tagged is False
with pytest.raises(ValueError, match='Cannot tag domain collection.*without a directory'):
collection.set_tag('News')
@pytest.mark.asyncio
async def test_tag_domain_collection_moves_files(
test_session, archive_factory, tag_factory, test_directory, make_files_structure, archive_directory,
await_switches,
):
"""Tagging domain collection with directory moves archive files."""
# Create domain collection with directory
# Use archive_directory fixture to match where archive_factory creates files
domain_dir = archive_directory / 'test.com'
domain_dir.mkdir(parents=True, exist_ok=True)
collection = Collection(
name='test.com',
kind='domain',
directory=domain_dir
)
test_session.add(collection)
test_session.flush()
# Create archives in domain directory
archive1 = archive_factory(domain='test.com')
archive2 = archive_factory(domain='test.com')
archive1.collection = collection
archive2.collection = collection
test_session.commit()
# Get file paths before move
old_paths = []
for archive in [archive1, archive2]:
for path in archive.my_paths():
old_paths.append(path)
assert all(p.is_file() for p in old_paths), "Files should exist before move"
# Create tag and assign to collection
tag = await tag_factory(name='Tech')
collection.set_tag('Tech')
test_session.commit()
# Compute new directory and move collection
new_directory = collection.format_directory('Tech')
# Create the destination directory before moving
new_directory.mkdir(parents=True, exist_ok=True)
await collection.move_collection(new_directory, test_session)
# Verify files moved
assert new_directory.is_dir(), "New directory should exist"
assert 'Tech' in str(new_directory), "New directory should contain tag name"
assert 'test.com' in str(new_directory), "New directory should contain domain name"
# Old paths should no longer exist
for old_path in old_paths:
assert not old_path.is_file(), f"Old file should be moved: {old_path}"
@pytest.mark.asyncio
async def test_domain_config_with_directory_and_tag(test_session, test_directory, tag_factory, async_client):
"""Domain config can include directory and tag_name."""
from modules.archive.lib import DomainsConfig
# Create config file
config_file = test_directory / 'domains.yaml'
config_file.write_text("""
collections:
- name: "example.com"
kind: "domain"
description: "News from example.com"
directory: "archives/example.com"
tag_name: "News"
""")
# Create tag first
tag = await tag_factory(name='News')
test_session.commit()
# Import config
domains_config = DomainsConfig()
domains_config.import_config(config_file)
# Verify collection created with directory and tag
collection = test_session.query(Collection).filter_by(
name='example.com',
kind='domain'
).one()
assert collection.directory is not None
assert 'example.com' in str(collection.directory)
assert collection.tag is not None
assert collection.tag.name == 'News'
assert collection.can_be_tagged is True
@pytest.mark.asyncio
async def test_domain_config_warns_tag_without_directory(test_session, test_directory, tag_factory, caplog,
async_client):
"""Config warns when tag_name provided without directory."""
from modules.archive.lib import DomainsConfig
# Create config file with tag but no directory
config_file = test_directory / 'domains.yaml'
config_file.write_text("""
collections:
- name: "example.com"
kind: "domain"
tag_name: "News"
""")
# Create tag first
tag = await tag_factory(name='News')
test_session.commit()
# Import config
domains_config = DomainsConfig()
domains_config.import_config(config_file)
# Verify collection created without tag
collection = test_session.query(Collection).filter_by(
name='example.com',
kind='domain'
).one()
assert collection.directory is None
assert collection.tag is None # Tag ignored
assert collection.can_be_tagged is False
# Check warning was logged
assert "tags require a directory" in caplog.text
@pytest.mark.asyncio
async def test_domain_config_export_includes_directory_and_tag(test_session, test_directory, tag_factory, async_client):
"""Exporting domain config includes directory and tag_name."""
from modules.archive.lib import DomainsConfig
from wrolpi.common import get_media_directory
import yaml
# Create domain collection with directory and tag
domain_dir = get_media_directory() / 'archives' / 'news.com'
domain_dir.mkdir(parents=True, exist_ok=True)
collection = Collection(
name='news.com',
kind='domain',
directory=domain_dir,
description='News website'
)
test_session.add(collection)
tag = await tag_factory(name='News')
collection.set_tag('News')
test_session.commit()
# Export config
config_file = test_directory / 'domains.yaml'
domains_config = DomainsConfig()
domains_config.dump_config(config_file, overwrite=True)
# Read and verify exported config
with open(config_file) as f:
exported = yaml.safe_load(f)
assert len(exported['collections']) == 1
domain_config = exported['collections'][0]
assert domain_config['name'] == 'news.com'
assert domain_config['kind'] == 'domain'
assert 'archives/news.com' in domain_config['directory']
assert domain_config['tag_name'] == 'News'
assert domain_config['description'] == 'News website'
@pytest.mark.asyncio
async def test_get_or_create_domain_collection_with_directory(test_session, test_directory):
"""get_or_create_domain_collection can create collection with directory."""
from modules.archive.lib import get_or_create_domain_collection
from wrolpi.common import get_media_directory
domain_dir = get_media_directory() / 'archives' / 'test.com'
domain_dir.mkdir(parents=True, exist_ok=True)
# Create with directory
collection = get_or_create_domain_collection(
test_session,
'https://test.com/article',
directory=domain_dir
)
assert collection.name == 'test.com'
assert collection.kind == 'domain'
assert collection.directory == domain_dir
assert collection.can_be_tagged is True
@pytest.mark.asyncio
async def test_get_archive_destination_with_directory(test_session, test_directory):
"""get_archive_destination returns collection directory when set."""
from modules.archive.lib import get_archive_destination
from wrolpi.common import get_media_directory
domain_dir = get_media_directory() / 'archives' / 'tagged' / 'example.com'
domain_dir.mkdir(parents=True, exist_ok=True)
collection = Collection(
name='example.com',
kind='domain',
directory=domain_dir
)
test_session.add(collection)
test_session.commit()
destination = get_archive_destination(collection)
assert destination == domain_dir
assert destination.is_dir()
@pytest.mark.asyncio
async def test_get_archive_destination_unrestricted(test_session):
"""get_archive_destination returns default path for unrestricted collection."""
from modules.archive.lib import get_archive_destination, get_archive_directory
collection = Collection(
name='example.com',
kind='domain',
directory=None # Unrestricted
)
test_session.add(collection)
test_session.commit()
destination = get_archive_destination(collection)
expected = get_archive_directory() / 'example.com'
assert destination == expected

283
modules/archive/test/test_lib.py
View file

@ -10,10 +10,11 @@ from PIL import Image
from pytz import utc
from modules.archive import lib
from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, model_archive_result, \
get_domains
from modules.archive.models import Archive, Domain
from modules.archive.lib import get_or_create_domain_collection, get_new_archive_files, delete_archives, \
model_archive_result, get_domains
from modules.archive.models import Archive
from wrolpi.api_utils import CustomJSONEncoder
from wrolpi.collections import Collection
from wrolpi.common import get_wrolpi_config
from wrolpi.db import get_db_session
from wrolpi.files import lib as files_lib
@ -37,7 +38,7 @@ def make_fake_archive_result(readability=True, screenshot=True, title=True):
@pytest.mark.asyncio
async def test_no_screenshot(test_directory, test_session):
async def test_no_screenshot(async_client, test_directory, test_session):
singlefile, readability, screenshot = make_fake_archive_result(screenshot=False)
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
assert isinstance(archive.singlefile_path, pathlib.Path)
@ -47,7 +48,7 @@ async def test_no_screenshot(test_directory, test_session):
@pytest.mark.asyncio
async def test_no_readability(test_directory, test_session):
async def test_no_readability(async_client, test_directory, test_session):
singlefile, readability, screenshot = make_fake_archive_result(readability=False)
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
assert isinstance(archive.singlefile_path, pathlib.Path)
@ -58,7 +59,7 @@ async def test_no_readability(test_directory, test_session):
@pytest.mark.asyncio
async def test_dict(test_session):
async def test_dict(async_client, test_session, test_directory):
singlefile, readability, screenshot = make_fake_archive_result()
d = (await model_archive_result('https://example.com', singlefile, readability, screenshot)).dict()
assert isinstance(d, dict)
@ -66,17 +67,18 @@ async def test_dict(test_session):
@pytest.mark.asyncio
async def test_relationships(test_session, example_singlefile):
async def test_relationships(async_client, test_session, example_singlefile):
with get_db_session(commit=True) as session:
url = 'https://wrolpi.org:443'
domain = get_or_create_domain(session, url)
collection = get_or_create_domain_collection(session, url)
archive = Archive.from_paths(test_session, example_singlefile)
archive.url = url
archive.domain_id = domain.id
archive.collection_id = collection.id
session.add(archive)
session.flush()
assert archive.domain == domain
assert archive.collection == collection
assert archive.domain == 'wrolpi.org'
@pytest.mark.asyncio
@ -119,8 +121,9 @@ async def test_archive_title(async_client, test_session, archive_factory, single
assert archive1.file_group.title is None
def test_archive_refresh_deleted_archive(test_client, test_session, archive_directory, archive_factory):
"""Archives/Domains should be deleted when archive files are deleted."""
@pytest.mark.asyncio
async def test_archive_refresh_deleted_archive(async_client, test_session, archive_directory, archive_factory):
"""Archives/domain collections should be deleted when archive files are deleted."""
archive1 = archive_factory('example.com', 'https://example.com/1')
archive2 = archive_factory('example.com', 'https://example.com/1')
archive3 = archive_factory('example.com')
@ -133,29 +136,30 @@ def test_archive_refresh_deleted_archive(test_client, test_session, archive_dire
def check_counts(archive_count, domain_count):
assert test_session.query(Archive).count() == archive_count, 'Archive count does not match'
assert test_session.query(Domain).count() == domain_count, 'Domain count does not match'
assert test_session.query(Collection).filter_by(
kind='domain').count() == domain_count, 'domain collection count does not match'
# All 5 archives are already in the DB.
check_counts(archive_count=5, domain_count=2)
test_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
check_counts(archive_count=5, domain_count=1)
# Delete archive2's files, it's the latest for 'https://example.com/1'
for path in archive2.my_paths():
path.unlink()
test_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
check_counts(archive_count=4, domain_count=1)
# Delete archive1's files, now the URL is empty.
for path in archive1.my_paths():
path.unlink()
test_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
check_counts(archive_count=3, domain_count=0)
# Delete archive3, now there is now example.com domain
for path in archive3.my_paths():
path.unlink()
test_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
check_counts(archive_count=2, domain_count=0)
# Delete all the rest of the archives
@ -163,12 +167,12 @@ def test_archive_refresh_deleted_archive(test_client, test_session, archive_dire
path.unlink()
for path in archive5.my_paths():
path.unlink()
test_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
check_counts(archive_count=0, domain_count=0)
@pytest.mark.asyncio
async def test_fills_contents_with_refresh(test_session, archive_factory, singlefile_contents_factory):
async def test_fills_contents_with_refresh(async_client, test_session, archive_factory, singlefile_contents_factory):
"""Refreshing archives fills in any missing contents."""
archive1 = archive_factory('example.com', 'https://example.com/one')
archive2 = archive_factory('example.com', 'https://example.com/one')
@ -212,7 +216,8 @@ async def test_fills_contents_with_refresh(test_session, archive_factory, single
assert archive3.file_group.title == 'last title' # from singlefile HTML
def test_delete_archive(test_session, archive_factory):
@pytest.mark.asyncio
async def test_delete_archive(async_client, test_session, archive_factory):
"""Archives can be deleted."""
archive1 = archive_factory('example.com', 'https://example.com/1')
archive2 = archive_factory('example.com', 'https://example.com/1')
@ -227,30 +232,35 @@ def test_delete_archive(test_session, archive_factory):
assert test_session.query(Archive).count() == 3
assert test_session.query(FileGroup).count() == 3
# Save paths before deletion (archives become detached after delete)
archive1_paths = list(archive1.my_paths())
archive2_paths = list(archive2.my_paths())
archive3_paths = list(archive3.my_paths())
# Delete the oldest.
delete_archives(archive1.id, archive3.id)
assert test_session.query(Archive).count() == 1
assert test_session.query(FileGroup).count() == 1
# Files were deleted.
assert archive1.my_paths() and not any(i.is_file() for i in archive1.my_paths())
assert archive3.my_paths() and not any(i.is_file() for i in archive3.my_paths())
assert archive1_paths and not any(i.is_file() for i in archive1_paths)
assert archive3_paths and not any(i.is_file() for i in archive3_paths)
# Archive2 is untouched
assert archive2.my_paths() and all(i.is_file() for i in archive2.my_paths())
assert archive2_paths and all(i.is_file() for i in archive2_paths)
# Delete the last archive. The Domain should also be deleted.
# Delete the last archive. The domain collection should also be deleted.
delete_archives(archive2.id)
assert test_session.query(Archive).count() == 0
domain = test_session.query(Domain).one_or_none()
domain = test_session.query(Collection).filter_by(kind='domain').one_or_none()
assert domain is None
# All Files were deleted.
assert test_session.query(FileGroup).count() == 0
assert archive2.my_paths() and not any(i.is_file() for i in archive2.my_paths())
assert archive2_paths and not any(i.is_file() for i in archive2_paths)
def test_get_domains(test_session, archive_factory):
"""
`get_domains` gets only Domains with Archives.
`get_domains` gets only domain collections with Archives.
"""
archive1 = archive_factory('example.com')
archive2 = archive_factory('example.com')
@ -316,7 +326,7 @@ async def test_new_archive(test_session, test_directory, fake_now):
fake_now(datetime(2000, 1, 2))
archive2 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
# Domain is reused.
# domain collection is reused.
assert archive1.domain == archive2.domain
@ -583,7 +593,7 @@ async def test_archive_meta(async_client, test_session, make_files_structure):
@pytest.mark.asyncio
async def test_refresh_archives_deleted_singlefile(test_session, make_files_structure, singlefile_contents_factory):
async def test_refresh_archives_deleted_singlefile(async_client, test_session, make_files_structure, singlefile_contents_factory):
"""Removing a Singlefile file from a FileGroup makes that group no longer an Archive."""
singlefile, readability = make_files_structure({
'2022-09-04-16-20-11_The Title.html': singlefile_contents_factory(),
@ -851,3 +861,218 @@ async def test_get_custom_archive_directory(async_client, test_directory, test_w
assert lib.get_archive_directory() == (test_directory / 'custom/archives')
assert lib.get_domain_directory('https://example.com') == (test_directory / 'custom/archives/example.com')
@pytest.mark.asyncio
async def test_detect_domain_directory_single_archive(async_client, test_directory, test_session, archive_factory):
"""detect_domain_directory should detect directory when all archives are in same location."""
from modules.archive.lib import detect_domain_directory
# Create an archive using the factory (which creates a domain collection and auto-detects directory)
archive = archive_factory(domain='example.com', url='https://example.com/page1')
test_session.flush()
# Get the collection
collection = archive.collection
assert collection is not None
assert collection.name == 'example.com'
# Directory should have been auto-detected during collection creation
assert collection.directory is not None
assert 'example.com' in str(collection.directory)
# Verify detect_domain_directory also returns the same result
detected = detect_domain_directory(collection, test_session)
assert detected is not None
assert str(detected) == 'archive/example.com'
@pytest.mark.asyncio
async def test_detect_domain_directory_multiple_archives(async_client, test_directory, test_session, archive_factory):
"""detect_domain_directory should detect common directory for multiple archives."""
from modules.archive.lib import detect_domain_directory
# Create multiple archives in the same domain
archive1 = archive_factory(domain='test.com', url='https://test.com/page1')
archive2 = archive_factory(domain='test.com', url='https://test.com/page2')
test_session.flush()
# Get the collection
collection = archive1.collection
assert collection is not None
assert collection.name == 'test.com'
# Detect directory
detected = detect_domain_directory(collection, test_session)
assert detected is not None
assert str(detected) == 'archive/test.com'
def test_detect_domain_directory_no_archives(test_directory, test_session):
"""detect_domain_directory should return None when collection has no archives."""
from modules.archive.lib import detect_domain_directory
# Create a domain collection without archives
collection = Collection(name='empty.com', kind='domain', directory=None)
test_session.add(collection)
test_session.flush()
# Detect directory - should return None
detected = detect_domain_directory(collection, test_session)
assert detected is None
@pytest.mark.asyncio
async def test_get_or_create_domain_collection_auto_detects_directory(async_client, test_directory, test_session, archive_factory):
"""get_or_create_domain_collection should auto-detect directory for existing archives."""
from modules.archive.lib import get_or_create_domain_collection
# Create an archive first
archive = archive_factory(domain='autodetect.com', url='https://autodetect.com/page1')
test_session.commit()
# The collection should have been created with no directory initially
# Call get_or_create_domain_collection again - should auto-detect
collection = get_or_create_domain_collection(test_session, 'https://autodetect.com/page2')
assert collection.directory is not None
assert 'autodetect.com' in str(collection.directory)
@pytest.mark.asyncio
async def test_update_domain_directories(async_client, test_directory, test_session, archive_factory):
"""update_domain_directories should fix existing collections that lost their directory."""
from modules.archive.lib import update_domain_directories
# Create an archive - this creates a collection WITH directory (auto-detected)
archive = archive_factory(domain='needsdir.com', url='https://needsdir.com/page1')
test_session.commit()
# Manually clear the directory to simulate legacy data
collection = archive.collection
collection.directory = None
test_session.commit()
# Verify collection has no directory
assert collection.directory is None
# Run update
count = update_domain_directories(test_session)
assert count == 1
# Verify directory was set
test_session.expire(collection)
assert collection.directory is not None
assert 'needsdir.com' in str(collection.directory)
def test_collection_unique_name_kind_constraint(test_directory, test_session, archive_factory):
"""Collections should have unique (name, kind) combinations.
This prevents duplicate domain collections with the same name.
"""
from sqlalchemy.exc import IntegrityError
# Create an archive which creates a domain collection
archive = archive_factory(domain='uniquetest.com')
test_session.commit()
# Get the original collection
original_collection = archive.collection
assert original_collection is not None
# Attempting to create a duplicate domain collection should fail
duplicate_collection = Collection(name='uniquetest.com', kind='domain', directory=None)
test_session.add(duplicate_collection)
with pytest.raises(IntegrityError) as exc_info:
test_session.commit()
# Verify it's the unique constraint violation
assert 'uq_collection_name_kind' in str(exc_info.value)
test_session.rollback()
def test_search_archives_by_domain(test_directory, test_session, archive_factory):
"""search_archives should correctly filter by domain collection name."""
from modules.archive.lib import search_archives
# Create archives in different domains
archive1 = archive_factory(domain='searchtest.com')
archive2 = archive_factory(domain='searchtest.com')
archive3 = archive_factory(domain='other.com')
test_session.commit()
# Search for archives in searchtest.com
file_groups, total = search_archives(
search_str=None,
domain='searchtest.com',
limit=10,
offset=0,
order=None,
tag_names=None
)
# Should return only archives from searchtest.com
assert total == 2
assert len(file_groups) == 2
# Search for other.com
file_groups, total = search_archives(
search_str=None,
domain='other.com',
limit=10,
offset=0,
order=None,
tag_names=None
)
assert total == 1
assert len(file_groups) == 1
def test_link_domain_and_downloads(test_session, test_download_manager):
"""Test that downloads are linked to domain collections."""
from wrolpi.downloader import Download
from modules.archive.lib import link_domain_and_downloads
# Create a domain collection with a directory
collection = Collection(name='example.com', kind='domain', directory='archive/example.com')
test_session.add(collection)
test_session.commit()
# 1. Download with matching destination directory (exact match)
download1 = Download(
url='https://other.com/rss',
downloader='rss',
sub_downloader='archive',
frequency=86400,
settings={'destination': 'archive/example.com'}
)
# 2. Download with destination in subdirectory
download2 = Download(
url='https://other.com/rss2',
downloader='rss',
sub_downloader='archive',
frequency=86400,
settings={'destination': 'archive/example.com/2025/01'}
)
# 3. RSS download with archive sub_downloader (matches by URL domain)
download3 = Download(
url='https://example.com/feed.xml',
downloader='rss',
sub_downloader='archive',
frequency=86400
)
# 4. Download without frequency (should NOT be linked)
download4 = Download(url='https://example.com/once', downloader='archive')
test_session.add_all([download1, download2, download3, download4])
test_session.commit()
assert not any(d.collection_id for d in [download1, download2, download3, download4])
link_domain_and_downloads(test_session)
assert download1.collection_id == collection.id # matched by directory (exact)
assert download2.collection_id == collection.id # matched by subdirectory
assert download3.collection_id == collection.id # matched by URL domain
assert download4.collection_id is None # no frequency = no link

19
modules/archive/test/test_models.py
View file

@ -1,9 +1,9 @@
from datetime import datetime
import pathlib
from datetime import datetime
import pytest
from modules.archive import Domain, Archive
from modules.archive import Archive
from wrolpi.common import get_wrolpi_config
@ -13,23 +13,23 @@ async def test_archive_download_destination(async_client, test_session, test_dir
wrolpi_config = get_wrolpi_config()
archive_factory(domain='wrolpi.org')
domain = test_session.query(Domain).one()
archive = archive_factory(domain='wrolpi.org')
test_session.commit()
# Test the default download directory.
assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org')
assert str(archive.download_directory) == str(test_directory / 'archive/wrolpi.org')
# Year of download is supported.
wrolpi_config.archive_destination = 'archives/%(domain)s/%(year)s'
assert str(domain.download_directory) == str(test_directory / 'archives/wrolpi.org/2000')
assert str(archive.download_directory) == str(test_directory / 'archives/wrolpi.org/2000')
# More download date is supported.
wrolpi_config.archive_destination = 'archive/%(domain)s/%(year)s/%(month)s/%(day)s'
assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org/2000/1/2')
assert str(archive.download_directory) == str(test_directory / 'archive/wrolpi.org/2000/1/2')
@pytest.mark.asyncio
async def test_set_screenshot(test_session, archive_factory, image_bytes_factory):
async def test_set_screenshot(async_client, test_session, archive_factory, image_bytes_factory):
"""Test Archive.set_screenshot() method with various scenarios."""
# ========== Test 1: Successfully set screenshot on archive without one ==========
@ -84,7 +84,8 @@ async def test_set_screenshot(test_session, archive_factory, image_bytes_factory
assert archive.screenshot_path == original_screenshot
# ========== Test 3: Error when trying to set non-existent screenshot file ==========
archive_no_screenshot = archive_factory('example.com', 'https://example.com/test2', 'Test Archive 2', screenshot=False)
archive_no_screenshot = archive_factory('example.com', 'https://example.com/test2', 'Test Archive 2',
screenshot=False)
test_session.commit()
# Try to set a screenshot that doesn't exist

19
modules/inventory/test/test_api.py
View file

@ -7,7 +7,8 @@ from modules.inventory import Item
from modules.inventory.inventory import save_item
def test_delete_items(test_session, init_test_inventory, test_client):
@pytest.mark.asyncio
async def test_delete_items(test_session, init_test_inventory, async_client):
"""
Multiple Items can be deleted in a single request.
"""
@ -24,14 +25,15 @@ def test_delete_items(test_session, init_test_inventory, test_client):
save_item(init_test_inventory.id, item)
items = test_session.query(Item).filter(Item.brand != None).all()
request, response = test_client.delete(f'/api/inventory/item/{items[0].id},{items[1].id}')
request, response = await async_client.delete(f'/api/inventory/item/{items[0].id},{items[1].id}')
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
# Refresh all items. Verify they were deleted.
[test_session.refresh(i) for i in items]
assert all([i.deleted_at for i in items])
def test_delete_item(test_session, init_test_inventory, test_client):
@pytest.mark.asyncio
async def test_delete_item(test_session, init_test_inventory, async_client):
"""
An item can be deleted by itself.
"""
@ -48,22 +50,23 @@ def test_delete_item(test_session, init_test_inventory, test_client):
item = test_session.query(Item).filter(Item.brand != None).one()
assert not item.deleted_at
request, response = test_client.delete(f'/api/inventory/item/{item.id}')
request, response = await async_client.delete(f'/api/inventory/item/{item.id}')
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
test_session.refresh(item)
assert item.deleted_at
def test_item_api(test_session, init_test_inventory, test_client):
@pytest.mark.asyncio
async def test_item_api(test_session, init_test_inventory, async_client):
"""An Item can be added to an Inventory."""
inventory_id = init_test_inventory.id
item = {'brand': 'Salty', 'name': 'Salt', 'item_size': '1', 'unit': 'lbs', 'count': '5',
'category': 'cooking ingredients', 'subcategory': 'salt', 'expiration_date': None}
request, response = test_client.post(f'/api/inventory/{inventory_id}/item', content=json.dumps(item))
request, response = await async_client.post(f'/api/inventory/{inventory_id}/item', content=json.dumps(item))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
# Get the Item we just created.
request, response = test_client.get(f'/api/inventory/{inventory_id}/item')
request, response = await async_client.get(f'/api/inventory/{inventory_id}/item')
assert response.status_code == HTTPStatus.OK
# assert item == response.json['items'][0]
item = response.json['items'][0]
@ -71,7 +74,7 @@ def test_item_api(test_session, init_test_inventory, test_client):
expiration_dates = ('1969-12-31T20:25:45.123450', None)
for expiration_date in expiration_dates:
item['expiration_date'] = expiration_date
request, response = test_client.put(f'/api/inventory/item/{item["id"]}', content=json.dumps(item))
request, response = await async_client.put(f'/api/inventory/item/{item["id"]}', content=json.dumps(item))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json

42
modules/map/test/test_api.py
View file

@ -3,10 +3,11 @@ from http import HTTPStatus
from itertools import zip_longest
import mock
import pytest
def assert_map_file_status(test_client, expected, status_code=HTTPStatus.OK):
request, response = test_client.get('/api/map/files')
async def assert_map_file_status(async_client, expected, status_code=HTTPStatus.OK):
request, response = await async_client.get('/api/map/files')
assert response.status_code == status_code
for pbf, expected_ in zip_longest(response.json['files'], expected):
imported = 'to be imported' if expected_['imported'] else 'to NOT be imported'
@ -16,9 +17,10 @@ def assert_map_file_status(test_client, expected, status_code=HTTPStatus.OK):
assert response.json['pending'] is None
@pytest.mark.asyncio
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_status_and_import(test_client, test_session, make_files_structure, mock_run_command):
async def test_status_and_import(async_client, test_session, make_files_structure, mock_run_command):
"""PBF files can be imported, and the status of the import can be monitored."""
pbf1, pbf2, dump1 = make_files_structure([
'map/pbf/some-country.osm.pbf',
@ -32,10 +34,10 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': False, 'path': 'map/pbf/some-country.osm.pbf'},
]
assert_map_file_status(test_client, expected)
await assert_map_file_status(async_client, expected)
body = {'files': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Can import multiple pbf files.
@ -44,10 +46,10 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
{'imported': True, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
assert_map_file_status(test_client, expected)
await assert_map_file_status(async_client, expected)
body = {'files': [str(pbf1), ], }
request, response = test_client.post('/api/map/import', content=json.dumps(body))
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Importing one pbf resets the import statuses.
@ -56,10 +58,10 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
assert_map_file_status(test_client, expected)
await assert_map_file_status(async_client, expected)
body = {'files': [str(dump1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Importing a dump also resets statuses.
@ -68,12 +70,13 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
assert_map_file_status(test_client, expected)
await assert_map_file_status(async_client, expected)
@pytest.mark.asyncio
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_run_command):
async def test_multiple_import(async_client, test_session, test_directory, make_files_structure, mock_run_command):
"""Multiple PBFs can be imported. Importing a second time overwrites the previous imports."""
pbf1, pbf2, pbf3, dump = make_files_structure([
'map/pbf/country1.osm.pbf',
@ -83,7 +86,7 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
])
body = {'files': [str(pbf2), str(pbf1)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Two PBFs can be merged and imported.
@ -93,10 +96,10 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
{'imported': True, 'path': 'map/pbf/country2.osm.pbf'},
{'imported': False, 'path': 'map/pbf/country3.osm.pbf'},
]
assert_map_file_status(test_client, expected)
await assert_map_file_status(async_client, expected)
body = {'files': [str(pbf3), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# New imports overwrite old imports.
@ -106,11 +109,11 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
{'imported': False, 'path': 'map/pbf/country2.osm.pbf'},
{'imported': True, 'path': 'map/pbf/country3.osm.pbf'},
]
assert_map_file_status(test_client, expected)
await assert_map_file_status(async_client, expected)
# Can't mix PBF and dump.
body = {'files': [str(dump), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Dump does not overwrite.
@ -120,12 +123,13 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
{'imported': False, 'path': 'map/pbf/country2.osm.pbf'},
{'imported': True, 'path': 'map/pbf/country3.osm.pbf'},
]
assert_map_file_status(test_client, expected)
await assert_map_file_status(async_client, expected)
def test_empty_import(test_client, test_session, test_directory):
@pytest.mark.asyncio
async def test_empty_import(async_client, test_session, test_directory):
"""Some files must be requested."""
body = {'files': []}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'validate' in response.json['message']

17
modules/otp/test/test_api.py
View file

@ -1,10 +1,13 @@
import json
from http import HTTPStatus
import pytest
def test_encrypt(test_client):
@pytest.mark.asyncio
async def test_encrypt(async_client):
body = dict(otp='asdf', plaintext='asdf')
request, response = test_client.post('/api/otp/encrypt_otp', content=json.dumps(body))
request, response = await async_client.post('/api/otp/encrypt_otp', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json == dict(
ciphertext='AAGK',
@ -13,9 +16,10 @@ def test_encrypt(test_client):
)
def test_decrypt(test_client):
@pytest.mark.asyncio
async def test_decrypt(async_client):
body = dict(otp='asdf', ciphertext='aagk')
request, response = test_client.post('/api/otp/decrypt_otp', content=json.dumps(body))
request, response = await async_client.post('/api/otp/decrypt_otp', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json == dict(
ciphertext='AAGK',
@ -24,6 +28,7 @@ def test_decrypt(test_client):
)
def test_get_html(test_client):
request, response = test_client.get('/api/otp/html')
@pytest.mark.asyncio
async def test_get_html(async_client):
request, response = await async_client.get('/api/otp/html')
assert response.status_code == HTTPStatus.OK

27
modules/videos/__init__.py
View file

@ -75,18 +75,19 @@ def video_cleanup():
with get_db_curs(commit=True) as curs:
# Delete all Videos if the FileModel no longer contains a video.
curs.execute('''
WITH deleted AS
(UPDATE file_group SET model=null WHERE model='video' AND mimetype NOT LIKE 'video/%' RETURNING id)
DELETE FROM video WHERE file_group_id = ANY(select id from deleted)
''')
WITH deleted AS
(UPDATE file_group SET model = null WHERE model = 'video' AND mimetype NOT LIKE 'video/%' RETURNING id)
DELETE
FROM video
WHERE file_group_id = ANY (select id from deleted)
''')
# Claim all Videos in a Channel's directory for that Channel. But, only if they have not yet been claimed.
curs.execute('''
UPDATE video v
SET
channel_id = c.id
FROM channel c
LEFT JOIN file_group fg ON fg.primary_path LIKE c.directory || '/%'::VARCHAR
WHERE
v.channel_id IS NULL
AND fg.id = v.file_group_id
''')
UPDATE video v
SET channel_id = c.id
FROM channel c
INNER JOIN collection col ON col.id = c.collection_id
LEFT JOIN file_group fg ON fg.primary_path LIKE col.directory || '/%'::VARCHAR
WHERE v.channel_id IS NULL
AND fg.id = v.file_group_id
''')

59
modules/videos/channel/lib.py
View file

@ -1,16 +1,18 @@
import pathlib
from pathlib import Path
from typing import List, Dict, Union
from typing import List, Union
from sqlalchemy import or_, func, desc, asc
from sqlalchemy.orm import Session
from sqlalchemy.orm import Session, joinedload
from wrolpi import flags
from wrolpi.collections import Collection
from wrolpi.common import logger, \
get_media_directory, wrol_mode_check, background_task
from wrolpi.db import get_db_curs, optional_session, get_db_session
from wrolpi.downloader import save_downloads_config, download_manager, Download
from wrolpi.errors import APIError, ValidationError, RefreshConflict
from wrolpi.tags import Tag
from wrolpi.vars import PYTEST
from .. import schema
from ..common import check_for_channel_conflicts
@ -30,15 +32,16 @@ async def get_minimal_channels() -> List[dict]:
# one that will consume the most resources.
stmt = '''
SELECT c.id,
c.name AS "name",
c.directory,
col.name AS "name",
col.directory,
c.url,
t.name AS "tag_name",
t.name AS "tag_name",
c.video_count,
c.total_size,
c.minimum_frequency
FROM channel AS c
LEFT JOIN tag t ON t.id = c.tag_id
INNER JOIN collection col ON col.id = c.collection_id
LEFT JOIN tag t ON t.id = col.tag_id
'''
curs.execute(stmt)
logger.debug(stmt)
@ -60,19 +63,22 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No
"""
channel: COD = None # noqa
# Try to find the channel by the most reliable methods first.
# Always eagerly load Collection to ensure name/directory/tag_id are available
if channel_id:
channel = session.query(Channel).filter_by(id=channel_id).one_or_none()
channel = session.query(Channel).options(joinedload(Channel.collection)).filter_by(id=channel_id).one_or_none()
if not channel and source_id:
channel = session.query(Channel).filter_by(source_id=source_id).one_or_none()
channel = session.query(Channel).options(joinedload(Channel.collection)).filter_by(
source_id=source_id).one_or_none()
if not channel and url:
channel = session.query(Channel).filter_by(url=url).one_or_none()
channel = session.query(Channel).options(joinedload(Channel.collection)).filter_by(url=url).one_or_none()
if not channel and directory:
directory = Path(directory)
if not directory.is_absolute():
directory = get_media_directory() / directory
channel = session.query(Channel).filter_by(directory=directory).one_or_none()
channel = session.query(Channel).join(Collection).filter(
Collection.directory == str(directory)).one_or_none()
if not channel and name:
channel = session.query(Channel).filter_by(name=name).one_or_none()
channel = session.query(Channel).join(Collection).filter(Collection.name == name).one_or_none()
if not channel:
raise UnknownChannel(f'No channel matches {channel_id=} {source_id=} {url=} {directory=}')
@ -147,7 +153,17 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
directory = get_media_directory() / directory if not directory.is_absolute() else directory
directory.mkdir(parents=True, exist_ok=True)
channel = Channel()
# Create Collection first
collection = Collection(
name=data.name,
kind='channel',
directory=directory,
)
session.add(collection)
session.flush([collection])
# Create Channel linked to Collection
channel = Channel(collection_id=collection.id)
session.add(channel)
session.flush([channel])
# Apply the changes now that we've OK'd them
@ -179,22 +195,24 @@ async def search_channels_by_name(name: str, limit: int = 5, session: Session =
name_no_spaces = ''.join(name.split(' '))
if order_by_video_count:
stmt = session.query(Channel, func.count(Video.id).label('video_count')) \
.join(Collection) \
.filter(or_(
Channel.name.ilike(f'%{name}%'),
Channel.name.ilike(f'%{name_no_spaces}%'),
Collection.name.ilike(f'%{name}%'),
Collection.name.ilike(f'%{name_no_spaces}%'),
)) \
.outerjoin(Video, Video.channel_id == Channel.id) \
.group_by(Channel.id, Channel.name) \
.order_by(desc('video_count'), asc(Channel.name)) \
.group_by(Channel.id, Collection.id, Collection.name) \
.order_by(desc('video_count'), asc(Collection.name)) \
.limit(limit)
channels = [i[0] for i in stmt]
else:
stmt = session.query(Channel) \
.join(Collection) \
.filter(or_(
Channel.name.ilike(f'%{name}%'),
Channel.name.ilike(f'%{name_no_spaces}%'),
Collection.name.ilike(f'%{name}%'),
Collection.name.ilike(f'%{name_no_spaces}%'),
)) \
.order_by(asc(Channel.name)) \
.order_by(asc(Collection.name)) \
.limit(limit)
channels = stmt.all()
return channels
@ -270,6 +288,5 @@ async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, chan
@optional_session
async def search_channels(tag_names: List[str], session: Session) -> List[Channel]:
"""Search Tagged Channels."""
from wrolpi.tags import Tag
channels = session.query(Channel).join(Tag).filter(Tag.name.in_(tag_names)).all()
channels = session.query(Channel).join(Collection).join(Tag).filter(Tag.name.in_(tag_names)).all()
return channels

74
modules/videos/channel/test/test_api.py
View file

@ -21,16 +21,18 @@ from wrolpi.files.models import Directory
from wrolpi.test.common import assert_dict_contains
def test_get_channels(test_directory, channel_factory, test_client):
@pytest.mark.asyncio
async def test_get_channels(test_directory, channel_factory, async_client):
channel_factory()
channel_factory()
channel_factory()
request, response = test_client.get('/api/videos/channels')
request, response = await async_client.get('/api/videos/channels')
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 3
def test_get_video(test_client, test_session, simple_channel, video_factory):
@pytest.mark.asyncio
async def test_get_video(async_client, test_session, simple_channel, video_factory):
"""Test that you get can information about a video. Test that video file can be gotten."""
now_ = now()
video1 = video_factory(channel_id=simple_channel.id, title='vid1')
@ -40,14 +42,14 @@ def test_get_video(test_client, test_session, simple_channel, video_factory):
test_session.commit()
# Test that a 404 is returned when no video exists
_, response = test_client.get('/api/videos/video/10')
_, response = await async_client.get('/api/videos/video/10')
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
assert response.json == {'code': 'UNKNOWN_VIDEO',
'error': 'Cannot find Video with id 10',
'message': 'The video could not be found.'}
# Get the video info we inserted
_, response = test_client.get('/api/videos/video/1')
_, response = await async_client.get('/api/videos/video/1')
assert response.status_code == HTTPStatus.OK, response.json
assert_dict_contains(response.json['file_group'], {'title': 'vid1'})
@ -92,9 +94,10 @@ async def test_channel_no_download_frequency(async_client, test_session, test_di
assert test_session.query(Channel).count() == 1
def test_video_file_name(test_session, simple_video, test_client):
@pytest.mark.asyncio
async def test_video_file_name(test_session, simple_video, async_client):
"""If a Video has no title, the front-end can use the file name as the title."""
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
_, resp = await async_client.get(f'/api/videos/video/{simple_video.id}')
assert resp.status_code == HTTPStatus.OK
assert resp.json['file_group']['video']['video_path'] == 'simple_video.mp4'
assert resp.json['file_group']['video'].get('stem') == 'simple_video'
@ -165,20 +168,21 @@ async def test_channel_conflicts(async_client, test_session, test_directory):
'message': 'Could not validate the contents of the request'}
def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory):
@pytest.mark.asyncio
async def test_channel_empty_url_doesnt_conflict(async_client, test_session, test_directory):
"""Two channels with empty URLs shouldn't conflict"""
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = dict(name='Fooz', directory=channel_directory)
request, response = test_client.post('/api/videos/channels', json=new_channel)
request, response = await async_client.post('/api/videos/channels', json=new_channel)
assert response.status_code == HTTPStatus.CREATED, response.json
location = response.headers['Location']
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory2).mkdir()
new_channel = dict(name='Barz', directory=channel_directory2)
request, response = test_client.post('/api/videos/channels', json=new_channel)
request, response = await async_client.post('/api/videos/channels', json=new_channel)
assert response.status_code == HTTPStatus.CREATED, response.json
assert location != response.headers['Location']
@ -235,11 +239,12 @@ async def test_channel_download_requires_refresh(
assert list(events_history) == []
def test_channel_post_directory(test_session, test_client, test_directory):
@pytest.mark.asyncio
async def test_channel_post_directory(test_session, async_client, test_directory):
"""A Channel can be created with or without an existing directory."""
# Channel can be created with a directory which is not on disk.
data = dict(name='foo', directory='foo')
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(data))
assert response.status_code == HTTPStatus.CREATED
directory = test_session.query(Channel).filter_by(id=1).one().directory
assert (test_directory / 'foo') == directory
@ -247,8 +252,9 @@ def test_channel_post_directory(test_session, test_client, test_directory):
assert directory.is_absolute()
def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}')
@pytest.mark.asyncio
async def test_channel_by_id(test_session, async_client, simple_channel, simple_video):
request, response = await async_client.get(f'/api/videos/channels/{simple_channel.id}')
assert response.status_code == HTTPStatus.OK
@ -334,10 +340,20 @@ async def test_change_channel_url(async_client, test_session, test_download_mana
assert channel.url == 'https://example.com/new-url', "Channel's URL was not changed."
def test_search_videos_channel(test_client, test_session, video_factory):
@pytest.mark.asyncio
async def test_search_videos_channel(async_client, test_session, video_factory, test_directory):
from wrolpi.collections import Collection
with get_db_session(commit=True) as session:
channel1 = Channel(name='Foo')
channel2 = Channel(name='Bar')
# Create Collections first
collection1 = Collection(name='Foo', kind='channel', directory=test_directory / 'foo')
collection2 = Collection(name='Bar', kind='channel', directory=test_directory / 'bar')
session.add_all([collection1, collection2])
session.flush()
# Create Channels linked to Collections
channel1 = Channel(collection_id=collection1.id)
channel2 = Channel(collection_id=collection2.id)
session.add(channel1)
session.add(channel2)
session.flush()
@ -346,7 +362,7 @@ def test_search_videos_channel(test_client, test_session, video_factory):
# Channels don't have videos yet
d = dict(channel_id=channel1.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
request, response = await async_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['file_groups']) == 0
@ -357,19 +373,19 @@ def test_search_videos_channel(test_client, test_session, video_factory):
session.add(vid2)
# Videos are gotten by their respective channels
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
request, response = await async_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['file_groups']) == 1
assert response.json['totals']['file_groups'] == 1
assert_dict_contains(response.json['file_groups'][0],
dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
dict(primary_path='foo/vid2.mp4', video=dict(channel_id=channel1.id)))
d = dict(channel_id=channel2.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
request, response = await async_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['file_groups']) == 1
assert_dict_contains(response.json['file_groups'][0],
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
dict(primary_path='bar/vid1.mp4', video=dict(channel_id=channel2.id)))
@pytest.mark.asyncio
@ -413,35 +429,37 @@ async def test_channel_download_id(async_client, test_session, tag_factory, simp
urls=['https://example.com/channel1'],
tag_names=[tag.name],
downloader=test_downloader.name,
settings=dict(channel_id=simple_channel.id),
settings=dict(),
collection_id=simple_channel.collection_id,
frequency=120,
)
request, response = await async_client.post(f'/api/download', json=body)
assert response.status_code == HTTPStatus.CREATED
download = test_session.query(Download).one()
assert download.url == 'https://example.com/channel1'
assert download.channel_id == simple_channel.id
assert download.collection_id == simple_channel.collection_id
# A Channel relationship can be removed from a Download.
# A Collection relationship can be removed from a Download.
body = dict(
urls=['https://example.com/channel1'],
tag_names=[tag.name],
downloader=test_downloader.name,
settings=dict(),
collection_id=None,
frequency=240,
)
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
assert download.channel_id is None
assert download.collection_id is None
assert download.frequency == 240
# A once-Download cannot be associated with a Channel.
# A once-Download cannot be associated with a Collection.
body = dict(
urls=['https://example.com/channel1'],
tag_names=[tag.name],
downloader=test_downloader.name,
settings=dict(channel_id=simple_channel.id),
collection_id=simple_channel.collection_id,
)
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST

12
modules/videos/channel/test/test_models.py
View file

@ -8,7 +8,7 @@ from modules.videos.models import Video
from wrolpi.downloader import Download, DownloadFrequency
def test_delete_channel_no_url(test_session, test_client, channel_factory):
def test_delete_channel_no_url(test_session, channel_factory):
"""
A Channel can be deleted even if it has no URL.
"""
@ -104,8 +104,16 @@ async def test_channel_download_relationships(test_session, download_channel):
def test_channel_info_json(test_session, test_directory):
channel = Channel(name='new channel')
from wrolpi.collections import Collection
# Create Collection first without directory
collection = Collection(name='new channel', kind='channel')
test_session.add(collection)
test_session.flush()
channel = Channel(collection_id=collection.id)
test_session.add(channel)
test_session.flush()
with pytest.raises(FileNotFoundError) as e:
# Cannot get info json because directory is not defined.

40
modules/videos/common.py
View file

@ -20,7 +20,6 @@ from wrolpi.dates import seconds_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
from .errors import ChannelNameConflict, ChannelURLConflict, ChannelDirectoryConflict, ChannelSourceIdConflict
from .models import Channel
logger = logger.getChild(__name__)
@ -52,6 +51,8 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None,
"""
Search for any channels that conflict with the provided args, raise a relevant exception if any conflicts are found.
"""
from .models import Channel
if not any([id_, url, name, directory]):
raise Exception('Cannot search for channel with no arguments')
@ -68,11 +69,13 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None,
if list(conflicts):
raise ChannelURLConflict()
if name:
conflicts = base_where.filter(Channel.name == name)
from wrolpi.collections import Collection
conflicts = base_where.join(Collection).filter(Collection.name == name)
if list(conflicts):
raise ChannelNameConflict()
if directory:
conflicts = base_where.filter(Channel.directory == directory)
from wrolpi.collections import Collection
conflicts = base_where.join(Collection).filter(Collection.directory == directory)
if list(conflicts):
raise ChannelDirectoryConflict()
if source_id:
@ -269,6 +272,8 @@ def extract_video_duration(video_path: Path) -> Optional[int]:
async def update_view_counts_and_censored(channel_id: int):
"""Update view_count for all Videos in a channel using its info_json file. Also sets FileGroup.censored
if Video is no longer available on the Channel."""
from .models import Channel
with get_db_session() as session:
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
channel_name = channel.name
@ -284,13 +289,14 @@ async def update_view_counts_and_censored(channel_id: int):
with get_db_curs(commit=True) as curs:
# Update the view_count for each video.
stmt = '''
WITH source AS (select * from json_to_recordset(%s::json) as (id text, view_count int))
UPDATE video
SET view_count = s.view_count
FROM source as s
WHERE source_id=s.id AND channel_id=%s
RETURNING video.id AS updated_ids
'''
WITH source AS (select * from json_to_recordset(%s::json) as (id text, view_count int))
UPDATE video
SET view_count = s.view_count
FROM source as s
WHERE source_id = s.id
AND channel_id = %s
RETURNING video.id AS updated_ids \
'''
curs.execute(stmt, (view_counts_str, channel_id))
count = len(curs.fetchall())
logger.info(f'Updated {count} view counts in DB for {channel_name}.')
@ -299,13 +305,13 @@ async def update_view_counts_and_censored(channel_id: int):
with get_db_curs(commit=True) as curs:
# Set FileGroup.censored if the video is no longer on the Channel.
stmt = '''
UPDATE file_group fg
SET censored = NOT (v.source_id = ANY(%(source_ids)s))
FROM video v
WHERE v.file_group_id = fg.id
AND v.channel_id = %(channel_id)s
RETURNING fg.id, fg.censored
'''
UPDATE file_group fg
SET censored = NOT (v.source_id = ANY (%(source_ids)s))
FROM video v
WHERE v.file_group_id = fg.id
AND v.channel_id = %(channel_id)s
RETURNING fg.id, fg.censored \
'''
curs.execute(stmt, {'channel_id': channel_id, 'source_ids': source_ids})
censored = len([i for i in curs.fetchall() if i['censored']])
logger.info(f'Set {censored} censored videos for {channel_name}.')

63
modules/videos/conftest.py
View file

@ -22,11 +22,23 @@ from wrolpi.vars import PROJECT_DIR
@pytest.fixture
def simple_channel(test_session, test_directory) -> Channel:
"""Get a Channel with the minimum properties. This Channel has no download!"""
channel = Channel(
directory=test_directory,
from wrolpi.collections import Collection
# Create Collection first
collection = Collection(
name='Simple Channel',
kind='channel',
directory=test_directory,
)
test_session.add(collection)
test_session.flush([collection])
# Create Channel linked to Collection
channel = Channel(
collection_id=collection.id,
url='https://example.com/channel1',
)
test_session.add(channel)
test_session.commit()
return channel
@ -36,21 +48,32 @@ def simple_channel(test_session, test_directory) -> Channel:
def channel_factory(test_session, test_directory):
"""Create a random Channel with a directory, and Download."""
from wrolpi.tags import Tag
from wrolpi.collections import Collection
def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None,
directory: pathlib.Path = None, tag_name: str = None):
name = name or str(uuid4())
tag = Tag.find_by_name(tag_name) if tag_name else None
channel = Channel(
tag_name = tag.name if tag else None
directory = directory or format_videos_destination(name, tag_name, url or f'https://example.com/{name}')
directory.mkdir(exist_ok=True, parents=True)
# Create Collection first
collection = Collection(
name=name,
url=url or f'https://example.com/{name}',
source_id=source_id,
tag=tag,
kind='channel',
directory=directory,
tag_id=tag.id if tag else None,
)
tag_name = tag.name if tag else None
channel.directory = directory or format_videos_destination(name, tag_name, channel.url)
channel.directory.mkdir(exist_ok=True, parents=True)
test_session.add(collection)
test_session.flush([collection])
# Create Channel linked to Collection
channel = Channel(
collection_id=collection.id,
url=url or f'https://example.com/{name}',
source_id=source_id,
)
test_session.add(channel)
test_session.flush([channel])
test_session.add(Directory(path=channel.directory, name=channel.directory.name))
@ -68,11 +91,25 @@ def channel_factory(test_session, test_directory):
@pytest.fixture
def download_channel(test_session, test_directory, video_download_manager) -> Channel:
"""Get a test Channel that has a download frequency."""
# Add a frequency to the test channel, then give it a download.
channel = Channel(directory=test_directory, name='Download Channel', url='https://example.com/channel1',
source_id='channel1')
from wrolpi.collections import Collection
# Create Collection first
collection = Collection(
name='Download Channel',
kind='channel',
directory=test_directory,
)
test_session.add(collection)
test_session.flush([collection])
# Create Channel linked to Collection
channel = Channel(
collection_id=collection.id,
url='https://example.com/channel1',
source_id='channel1',
)
test_session.add(channel)
test_session.flush([channel, ])
test_session.flush([channel])
assert channel and channel.id and channel.url
download = channel.get_or_create_download(channel.url, 60, test_session)
assert download.url == channel.url

17
modules/videos/downloader.py
View file

@ -5,7 +5,6 @@ import logging
import os.path
import pathlib
import re
import sys
import traceback
from abc import ABC
from datetime import timedelta
@ -208,7 +207,7 @@ class ChannelDownloader(Downloader, ABC):
# The settings to send to the VideoDownloader.
settings = dict()
if channel:
settings.update(dict(channel_id=channel.id, channel_url=download.url))
settings.update(dict(collection_id=channel.collection_id, channel_url=download.url))
if download.destination:
destination = get_absolute_media_path(download.destination)
settings['destination'] = str(destination) # Need str for JSON conversion
@ -622,14 +621,20 @@ class VideoDownloader(Downloader, ABC):
except UnknownChannel:
# Destination must not be a channel.
pass
channel_id = download.channel_id
# Look up Channel via Collection or channel_url from settings
collection_id = download.collection_id
channel_url = settings.get('channel_url')
if not channel and (channel_id or channel_url):
if not channel and (collection_id or channel_url):
# Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
logger.info(f'Using download.settings to find channel')
try:
channel = get_channel(channel_id=channel_id, url=channel_url, return_dict=False)
logger.debug(f'Found channel with channel url {channel=}')
# Find Channel by collection_id or URL
if collection_id:
with get_db_session() as session:
channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
if not channel and channel_url:
channel = get_channel(url=channel_url, return_dict=False)
logger.debug(f'Found channel with collection_id or channel url {channel=}')
except UnknownChannel:
# We do not need a Channel if we have a destination directory.
if not destination:

66
modules/videos/lib.py
View file

@ -252,7 +252,6 @@ class ChannelsConfig(ConfigFile):
self.update({'channels': value})
def import_config(self, file: pathlib.Path = None, send_events=False):
from modules.videos.channel.lib import get_channel
super().import_config()
try:
channels = self.channels
@ -268,31 +267,8 @@ class ChannelsConfig(ConfigFile):
for option in (i for i in REQUIRED_OPTIONS if i not in data):
raise ConfigError(f'Channel "{directory}" is required to have "{option}"')
# Try to find Channel by directory because it is unique.
channel = Channel.get_by_path(directory, session)
if not channel:
try:
# Try to find Channel using other attributes before creating new Channel.
channel = get_channel(
session,
source_id=data.get('source_id'),
url=data.get('url'),
directory=str(directory),
return_dict=False,
)
except UnknownChannel:
# Channel not yet in the DB, add it.
channel = Channel(directory=directory)
session.add(channel)
channel.flush()
# TODO refresh the files in the channel.
logger.warning(f'Creating new Channel from config: {directory}')
# Copy existing channel data, update all values from the config. This is necessary to clear out
# values not in the config.
full_data = channel.dict()
full_data.update(data)
channel.update(full_data)
# Create or update Channel from config (handles Collection creation)
channel = Channel.from_config(data, session=session)
updated_channel_ids.add(channel.id)
if not channel.source_id and channel.url and flags.have_internet.is_set():
@ -519,8 +495,9 @@ def set_test_downloader_config(enabled: bool):
def get_channels_config_from_db(session: Session) -> dict:
"""Create a dictionary that contains all the Channels from the DB."""
channels = session.query(Channel).order_by(Channel.directory).all()
channels = sorted((i.config_view() for i in channels), key=lambda i: i['directory'])
from wrolpi.collections import Collection
channels = session.query(Channel).join(Collection).order_by(Collection.directory).all()
channels = sorted((i.config_view() for i in channels), key=lambda i: i['directory'] if i['directory'] else '')
return dict(channels=channels)
@ -572,9 +549,12 @@ def import_channels_config():
def link_channel_and_downloads(session: Session, channel_: Type[Base] = Channel, download_: Type[Base] = Download):
"""Create any missing Downloads for any Channel.url/Channel.directory that has a Download. Associate any Download
related to a Channel."""
# Only Downloads with a frequency can be a Channel Download.
"""Associate any Download related to a Channel via its Collection.
Downloads are linked to Collections (via collection_id) rather than directly to Channels.
This function finds Channels and links their Downloads to the Channel's Collection.
"""
# Only Downloads with a frequency can be a Collection Download.
downloads = list(session.query(download_).filter(download_.frequency.isnot(None)).all())
# Download.url is unique and cannot be null.
downloads_by_url = {i.url: i for i in downloads}
@ -586,26 +566,27 @@ def link_channel_and_downloads(session: Session, channel_: Type[Base] = Channel,
for channel in channels:
directory = str(channel.directory)
for download in downloads_with_destination:
if download.settings['destination'] == directory and not download.channel_id:
download.channel_id = channel.id
if download.settings['destination'] == directory and not download.collection_id:
download.collection_id = channel.collection_id
need_commit = True
download = downloads_by_url.get(channel.url)
if download and not download.channel_id:
download.channel_id = channel.id
if download and not download.collection_id:
download.collection_id = channel.collection_id
need_commit = True
# Get any Downloads for a Channel's RSS feed.
rss_url = channel.get_rss_url()
if rss_url and (download := downloads_by_url.get(rss_url)):
download.channel_id = channel.id
need_commit = True
if not download.collection_id:
download.collection_id = channel.collection_id
need_commit = True
# Associate any Download which shares a Channel's URL.
for download in downloads:
channel = channel_.get_by_url(download.url, session)
if channel and not download.channel_id:
download.channel_id = channel.id
if channel and not download.collection_id:
download.collection_id = channel.collection_id
need_commit = True
if need_commit:
@ -676,10 +657,11 @@ async def get_statistics():
if monthly_videos else 0
curs.execute('''
SELECT COUNT(c.id) AS "channels",
COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels"
SELECT COUNT(c.id) AS "channels",
COUNT(c.id) FILTER ( WHERE col.tag_id IS NOT NULL ) AS "tagged_channels"
FROM channel c
LEFT JOIN public.tag t on t.id = c.tag_id
LEFT JOIN collection col on col.id = c.collection_id
LEFT JOIN public.tag t on t.id = col.tag_id
''')
channel_stats = dict(curs.fetchone())
ret = dict(statistics=dict(

142
modules/videos/models.py
View file

@ -531,9 +531,8 @@ class Video(ModelHelper, Base):
class Channel(ModelHelper, Base):
__tablename__ = 'channel'
id = Column(Integer, primary_key=True)
name = Column(String)
# name and directory are stored in Collection, accessed via properties
url = Column(String, unique=True) # will only be downloaded if related Download exists.
directory: pathlib.Path = Column(MediaPathType)
generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow.
calculate_duration = Column(Boolean, default=True) # use ffmpeg to extract duration (slower than info json).
download_missing_data = Column(Boolean, default=True) # fetch missing data like `source_id` and video comments.
@ -549,16 +548,65 @@ class Channel(ModelHelper, Base):
info_date = Column(Date)
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id')
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'))
tag = relationship('Tag', primaryjoin='Channel.tag_id==Tag.id')
collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'))
collection = relationship('Collection', foreign_keys=[collection_id])
@property
def downloads(self) -> InstrumentedList:
"""Get downloads from associated Collection."""
return self.collection.downloads if self.collection else []
def __repr__(self):
return f'<Channel id={self.id} name={repr(self.name)} url={self.url} directory={self.directory}>'
@property
def name(self) -> str | None:
"""Delegate name to Collection"""
return self.collection.name if self.collection else None
@name.setter
def name(self, value: str):
"""Set name on Collection"""
if self.collection:
self.collection.name = value
@property
def directory(self) -> pathlib.Path | None:
"""Delegate directory to Collection"""
return self.collection.directory if self.collection else None
@directory.setter
def directory(self, value: pathlib.Path | str):
"""Set directory on Collection"""
if self.collection:
self.collection.directory = value
@property
def tag(self):
"""Delegate tag to Collection"""
return self.collection.tag if self.collection else None
@tag.setter
def tag(self, value):
"""Set tag on Collection"""
if self.collection:
self.collection.tag = value
@property
def tag_id(self) -> int | None:
"""Delegate tag_id to Collection"""
return self.collection.tag_id if self.collection else None
@tag_id.setter
def tag_id(self, value: int):
"""Set tag_id on Collection"""
if self.collection:
self.collection.tag_id = value
@property
def tag_name(self) -> str | None:
return self.tag.name if self.tag else None
"""Delegate tag_name to Collection"""
return self.collection.tag_name if self.collection else None
@property
def location(self) -> str:
@ -597,7 +645,7 @@ class Channel(ModelHelper, Base):
data = data.copy()
# URL should not be empty string.
url = data.pop('url')
url = data.pop('url', None)
self.url = url or None
session: Session = Session.object_session(self)
@ -637,7 +685,7 @@ class Channel(ModelHelper, Base):
continue
download = self.get_or_create_download(url, frequency, session=session, reset_attempts=True)
download.channel_id = channel_id
download.collection_id = self.collection_id
session.add(download)
def config_view(self) -> dict:
@ -657,13 +705,62 @@ class Channel(ModelHelper, Base):
)
return config
@classmethod
def from_config(cls, data: dict, session: Session = None) -> 'Channel':
"""
Create or update a Channel from config data. This also creates/updates the Collection.
Args:
data: Config dict containing channel metadata (name, directory, url, source_id, etc.)
session: Database session
Returns:
The created or updated Channel
"""
from wrolpi.collections import Collection
from wrolpi.db import get_db_session
if session is None:
session = get_db_session()
# Ensure this is treated as a channel collection
data = data.copy()
data['kind'] = 'channel'
# Create or update the Collection first
collection = Collection.from_config(data, session=session)
# Extract Channel-specific fields
url = data.get('url')
source_id = data.get('source_id')
# Find existing Channel by collection_id or create new one
channel = session.query(cls).filter_by(collection_id=collection.id).one_or_none()
if not channel:
# Create new Channel
channel = cls(
collection_id=collection.id,
url=url,
source_id=source_id,
)
session.add(channel)
session.flush([channel])
# Use the update() method to handle all fields including downloads
channel.update(data)
return channel
@staticmethod
def get_by_path(path: pathlib.Path, session: Session) -> Optional['Channel']:
if not path:
raise RuntimeError('Must provide path to get Channel')
path = pathlib.Path(path) if isinstance(path, str) else path
path = str(path.absolute()) if path.is_absolute() else str(get_media_directory() / path)
channel = session.query(Channel).filter_by(directory=path).one_or_none()
# Query through Collection relationship since directory is now on Collection
from wrolpi.collections import Collection
channel = session.query(Channel).join(Collection).filter(Collection.directory == path).one_or_none()
return channel
def __json__(self) -> dict:
@ -679,6 +776,8 @@ class Channel(ModelHelper, Base):
def dict(self, with_statistics: bool = False, with_downloads: bool = True) -> dict:
d = super(Channel, self).dict()
# Add Collection-delegated properties
d['name'] = self.name
d['tag_name'] = self.tag_name
d['rss_url'] = self.get_rss_url()
d['directory'] = \
@ -777,26 +876,17 @@ class Channel(ModelHelper, Base):
reset_attempts: bool = False) -> Download:
"""Get a Download record, if it does not exist, create it. Create a Download if necessary
which goes into this Channel's directory.
"""
if not isinstance(url, str) or not url:
raise InvalidDownload(f'Cannot get Download without url')
if not frequency:
raise InvalidDownload('Download for Channel must have a frequency')
Delegates to Collection.get_or_create_download() with Channel-specific downloaders.
"""
from modules.videos.downloader import ChannelDownloader, VideoDownloader
download = Download.get_by_url(url, session=session)
if not download:
download = download_manager.recurring_download(url, frequency, ChannelDownloader.name, session=session,
sub_downloader_name=VideoDownloader.name,
destination=self.directory,
reset_attempts=reset_attempts,
)
if reset_attempts:
download.attempts = 0
download.channel_id = self.id
return download
return self.collection.get_or_create_download(
url, frequency, session=session,
reset_attempts=reset_attempts,
downloader_name=ChannelDownloader.name,
sub_downloader_name=VideoDownloader.name
)
def get_rss_url(self) -> str | None:
"""Return the RSS Feed URL for this Channel, if any is possible."""

26
modules/videos/test/test_api.py
View file

@ -43,8 +43,9 @@ async def test_refresh_videos_index(async_client, test_session, test_directory,
assert not video.file_group.d_text, 'Video captions were not removed'
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory, video_file_factory,
image_bytes_factory):
@pytest.mark.asyncio
async def test_refresh_videos(async_client, test_session, test_directory, simple_channel, video_factory,
video_file_factory, image_bytes_factory):
subdir = test_directory / 'subdir'
subdir.mkdir()
@ -81,7 +82,7 @@ def test_refresh_videos(test_client, test_session, test_directory, simple_channe
stmt = "INSERT INTO video (file_group_id) values (%(video_id)s)"
curs.execute(stmt, {'video_id': str(video4_id)})
test_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
test_session.expire_all()
@ -104,12 +105,13 @@ def test_refresh_videos(test_client, test_session, test_directory, simple_channe
# Remove video1's poster, video1 should be updated.
video1_video_path = str(video1.video_path)
video1.poster_path.unlink()
test_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
video1 = Video.get_by_path(video1_video_path, test_session)
assert not video1.poster_path
def test_channels_with_videos(test_session, test_client, test_directory, channel_factory, video_factory):
@pytest.mark.asyncio
async def test_channels_with_videos(test_session, async_client, test_directory, channel_factory, video_factory):
channel1 = channel_factory('channel1', name='channel1')
channel2 = channel_factory('channel2', name='channel2')
vid1 = video_factory(channel_id=channel1.id, with_video_file=True)
@ -132,10 +134,12 @@ def test_channels_with_videos(test_session, test_client, test_directory, channel
assert vid2_path.is_file(), 'Video file was deleted'
assert vid3_path.is_file(), 'Video file was deleted'
test_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
assert test_session.query(Video).count() == 3, 'Did not find correct amount of video files.'
assert {i[0] for i in test_session.query(Channel.name)} == {'channel1', 'channel2'}, 'Channels were changed.'
from wrolpi.collections import Collection
assert {i[0] for i in test_session.query(Collection.name).join(Channel).filter(Collection.kind == 'channel')} == {
'channel1', 'channel2'}, 'Channels were changed.'
vid1 = Video.get_by_path(vid1_path)
vid2 = Video.get_by_path(vid2_path)
@ -146,11 +150,12 @@ def test_channels_with_videos(test_session, test_client, test_directory, channel
assert vid3.channel is None
def test_api_download(test_session, test_client, test_directory):
@pytest.mark.asyncio
async def test_api_download(test_session, async_client, test_directory):
"""A video can be downloaded."""
content = dict(urls=['https://example.com/video1', ], downloader='video', destination='dest',
settings=dict(excluded_urls='example.com'))
request, response = test_client.post('/api/download', content=json.dumps(content))
request, response = await async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
download = test_session.query(Download).one()
@ -364,7 +369,8 @@ async def test_video_file_format(async_client, test_session, fake_now):
@pytest.mark.asyncio
async def test_video_upload_file_tracking(test_session, async_client, video_factory, await_switches, make_multipart_form,
async def test_video_upload_file_tracking(test_session, async_client, video_factory, await_switches,
make_multipart_form,
image_bytes_factory):
"""Test that uploading info.json, poster, and caption files via /api/files/upload properly tracks them in Video and FileGroup."""
from wrolpi.common import get_relative_to_media_directory

23
modules/videos/test/test_common.py
View file

@ -257,6 +257,11 @@ async def test_import_channel_delete_missing_channels(await_switches, test_sessi
channel1 = channel_factory(source_id='foo')
channel2 = channel_factory(source_id='bar')
test_session.commit()
# Capture directory values before deletion (channels will be detached after deletion)
channel1_dir = str(channel1.directory)
channel2_dir = str(channel2.directory)
# Write Channels to the config file.
save_channels_config()
test_session.delete(channel1)
@ -267,29 +272,29 @@ async def test_import_channel_delete_missing_channels(await_switches, test_sessi
# Importing the config creates two Channels.
import_channels_config()
assert len(test_session.query(Channel).all()) == 2
assert str(channel1.directory) in test_channels_config.read_text()
assert str(channel2.directory) in test_channels_config.read_text()
assert channel1_dir in test_channels_config.read_text()
assert channel2_dir in test_channels_config.read_text()
# Delete channel2 from the config file.
config = get_channels_config()
config_dict = config.dict()
config_dict['channels'] = [i for i in config.channels if i['directory'] != str(channel2.directory)]
config_dict['channels'] = [i for i in config.channels if i['directory'] != channel2_dir]
config.update(config_dict)
await await_switches()
assert str(channel1.directory) in test_channels_config.read_text()
assert str(channel2.directory) not in test_channels_config.read_text()
assert channel1_dir in test_channels_config.read_text()
assert channel2_dir not in test_channels_config.read_text()
# Importing the config deletes the Channel record.
import_channels_config()
assert len(test_session.query(Channel).all()) == 1
assert str(channel1.directory) in test_channels_config.read_text()
assert str(channel2.directory) not in test_channels_config.read_text()
assert channel1_dir in test_channels_config.read_text()
assert channel2_dir not in test_channels_config.read_text()
# Saving and importing does not change anything.
save_channels_config()
import_channels_config()
assert str(channel1.directory) in test_channels_config.read_text()
assert str(channel2.directory) not in test_channels_config.read_text()
assert channel1_dir in test_channels_config.read_text()
assert channel2_dir not in test_channels_config.read_text()
@pytest.mark.asyncio

30
modules/videos/test/test_downloader.py
View file

@ -173,7 +173,7 @@ async def test_download_channel(test_session, test_directory, simple_channel, vi
downloads = video_download_manager.get_once_downloads(test_session)
assert {i.url for i in downloads} == {'https://youtube.com/watch?v=video_2_url'}
assert downloads[0].settings == {
'channel_id': 1,
'collection_id': simple_channel.collection_id,
'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
'destination': str(test_directory),
}
@ -191,7 +191,7 @@ async def test_download_channel(test_session, test_directory, simple_channel, vi
downloads = video_download_manager.get_once_downloads(test_session)
assert {i.url for i in downloads} == {'https://youtube.com/watch?v=video_1_url'}
assert downloads[0].settings == {
'channel_id': 1,
'collection_id': simple_channel.collection_id,
'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
'destination': str(test_directory),
}
@ -203,13 +203,23 @@ async def test_get_or_create_channel(async_client, test_session, test_directory,
Attempt to use an existing Channel if we can match it.
"""
from wrolpi.collections import Collection
one, two = await tag_factory(), await tag_factory()
c1 = Channel(name='foo', source_id='foo', url='https://example.com')
c2 = Channel(name='bar', source_id='bar')
c3 = Channel(name='baz', source_id='baz', url='https://example.net')
c4 = Channel(name='qux')
test_session.add_all([c1, c2, c3, c4])
# Create Collections for each Channel
def create_channel_with_collection(name, source_id=None, url=None):
collection = Collection(name=name, kind='channel')
test_session.add(collection)
test_session.flush([collection])
channel = Channel(collection_id=collection.id, source_id=source_id, url=url)
test_session.add(channel)
test_session.flush([channel])
return channel
c1 = create_channel_with_collection('foo', source_id='foo', url='https://example.com')
c2 = create_channel_with_collection('bar', source_id='bar')
c3 = create_channel_with_collection('baz', source_id='baz', url='https://example.net')
c4 = create_channel_with_collection('qux')
test_session.commit()
# All existing channels should be used.
@ -549,7 +559,8 @@ def test_normalize_video_file_names(test_directory, video_download_manager):
@pytest.mark.asyncio
async def test_video_download_cookies(test_session, test_directory, mock_video_extract_info, await_switches,
video_download_manager, mock_video_process_runner, image_file, test_videos_downloader_config):
video_download_manager, mock_video_process_runner, image_file,
test_videos_downloader_config):
config = get_videos_downloader_config()
config.browser_profile = str(test_directory / 'firefox/some directory')
@ -577,7 +588,8 @@ async def test_video_download_cookies(test_session, test_directory, mock_video_e
@pytest.mark.asyncio
async def test_video_download_always_use_cookies(test_session, test_directory, mock_video_extract_info, await_switches,
video_download_manager, mock_video_process_runner, image_file, test_videos_downloader_config):
video_download_manager, mock_video_process_runner, image_file,
test_videos_downloader_config):
config = get_videos_downloader_config()
config.always_use_browser_profile = True

16
modules/videos/test/test_lib.py
View file

@ -191,13 +191,13 @@ def test_link_channel_and_downloads(test_session, channel_factory, test_download
test_session.add_all([download1, download2, download3])
test_session.commit()
assert test_session.query(Download).count() == 3
assert not any(i.channel_id for i in test_download_manager.get_downloads(test_session))
assert not any(i.collection_id for i in test_download_manager.get_downloads(test_session))
# `link_channel_and_downloads` creates missing Downloads.
# `link_channel_and_downloads` links Downloads to Collections.
lib.link_channel_and_downloads(session=test_session)
assert test_session.query(Download).count() == 3
assert all(i.channel_id for i in test_download_manager.get_recurring_downloads(test_session))
assert not any(i.channel_id for i in test_download_manager.get_once_downloads(test_session))
assert all(i.collection_id for i in test_download_manager.get_recurring_downloads(test_session))
assert not any(i.collection_id for i in test_download_manager.get_once_downloads(test_session))
@pytest.mark.asyncio
@ -233,13 +233,13 @@ def test_link_channel_and_downloads_migration(async_client, test_session, channe
assert test_session.query(Download).count() == 4
d1, d2a, d2b, d2c = test_session.query(Download).order_by(Download.url).all()
assert d1.url == channel1.url == 'https://example.com/channel1' and d1.frequency == 1
assert d1.channel_id
assert d1.collection_id == channel1.collection_id
assert d2a.url == 'https://example.com/channel2' and d2a.frequency == 1
assert d2a.channel_id
assert d2a.collection_id == channel2.collection_id
assert d2b.url == 'https://example.com/channel2/rss' and d2b.frequency == 1
assert d2b.channel_id
assert d2b.collection_id == channel2.collection_id
assert d2c.url == 'https://example.com/channel2/video/1' and d2c.frequency is None
assert not d2c.channel_id
assert not d2c.collection_id
@pytest.mark.asyncio

11
modules/zim/lib.py
View file

@ -111,12 +111,11 @@ def get_all_entries_tags():
},
"""
stmt = '''
SELECT tz.zim_id, tz.zim_entry, array_agg(t.name)::TEXT[]
FROM
tag t
LEFT JOIN tag_zim tz on t.id = tz.tag_id
GROUP BY 1, 2
'''
SELECT tz.zim_id, tz.zim_entry, array_agg(t.name)::TEXT[]
FROM tag t
LEFT JOIN tag_zim tz on t.id = tz.tag_id
GROUP BY 1, 2 \
'''
with get_db_curs() as curs:
curs.execute(stmt)
entries = dict()

24
modules/zim/test/test_api.py
View file

@ -133,18 +133,19 @@ async def test_zim_tag_and_untag(async_client, test_session, test_zim, tag_facto
assert response.status_code == HTTPStatus.NOT_FOUND
def test_zim_subscribe(test_session, test_client):
@pytest.mark.asyncio
async def test_zim_subscribe(test_session, async_client):
"""A Kiwix subscription can be scheduled in the API. The language can be changed."""
# Subscribe to English.
content = dict(name='Wikipedia (mini)', language='en')
request, response = test_client.post('/api/zim/subscribe', content=json.dumps(content))
request, response = await async_client.post('/api/zim/subscribe', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
download: Download = test_session.query(Download).one()
assert download.url == 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_'
assert download.info_json == {'language': 'en', 'name': 'Wikipedia (mini)'}
assert download.frequency
request, response = test_client.get('/api/zim/subscribe')
request, response = await async_client.get('/api/zim/subscribe')
assert response.status_code == HTTPStatus.OK
assert response.json['subscriptions'] == {
'Wikipedia (mini)': {'download_id': 1,
@ -155,7 +156,7 @@ def test_zim_subscribe(test_session, test_client):
# Change subscription to French.
content = dict(name='Wikipedia (mini)', language='fr')
request, response = test_client.post('/api/zim/subscribe', content=json.dumps(content))
request, response = await async_client.post('/api/zim/subscribe', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(Download).count() == 1
@ -163,7 +164,7 @@ def test_zim_subscribe(test_session, test_client):
assert download.url == 'https://download.kiwix.org/zim/wikipedia/wikipedia_fr_all_mini_'
assert download.info_json == {'language': 'fr', 'name': 'Wikipedia (mini)'}
assert download.frequency
request, response = test_client.get('/api/zim/subscribe')
request, response = await async_client.get('/api/zim/subscribe')
assert response.status_code == HTTPStatus.OK
assert response.json['subscriptions'] == {
'Wikipedia (mini)': {'download_id': 1,
@ -173,9 +174,9 @@ def test_zim_subscribe(test_session, test_client):
'name': 'Wikipedia (mini)'}}
# Delete subscription.
request, response = test_client.delete('/api/zim/subscribe/1')
request, response = await async_client.delete('/api/zim/subscribe/1')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = test_client.get('/api/zim/subscribe')
request, response = await async_client.get('/api/zim/subscribe')
assert response.status_code == HTTPStatus.OK
assert response.json['subscriptions'] == {}
# No subscriptions or downloads.
@ -184,17 +185,18 @@ def test_zim_subscribe(test_session, test_client):
# English Stack Exchange is special.
content = dict(name='Stackoverflow (Stack Exchange)', language='en')
request, response = test_client.post('/api/zim/subscribe', content=json.dumps(content))
request, response = await async_client.post('/api/zim/subscribe', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
download: Download = test_session.query(Download).one()
assert download.url == 'https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_en_all_'
def test_get_zim_entry(test_session, test_client, test_zim):
request, response = test_client.get('/api/zim/1/entry/home')
@pytest.mark.asyncio
async def test_get_zim_entry(test_session, async_client, test_zim):
request, response = await async_client.get('/api/zim/1/entry/home')
assert response.status_code == HTTPStatus.OK
request, response = test_client.get('/api/zim/1/entry/does not exist')
request, response = await async_client.get('/api/zim/1/entry/does not exist')
assert response.status_code == HTTPStatus.NOT_FOUND