mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Remove unused code found by vulture (#885)
Deletes functions, methods, classes, schema dataclasses, error classes, Events senders, and module-level constants that nothing references, including tests. Also drops the unused `doc_cleanup(test_directory)` and `_wait_for_sub_job_with_progress(collection_idx)` parameters, and the imports that only served removed code. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a248104b2f
commit
22ada7eb57
34 changed files with 9 additions and 586 deletions
|
|
@ -26,7 +26,6 @@ from .lib import is_singlefile_file, request_archive, SINGLEFILE_HEADER, get_url
|
|||
SinglefileArtifacts, WrittenArchiveFiles, extract_singlefile_artifacts, register_archive, write_archive_files
|
||||
from .models import Archive
|
||||
|
||||
PRETTY_NAME = 'Archive'
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from wrolpi.common import get_media_directory, get_relative_to_media_directory,
|
|||
from wrolpi.collections.models import validate_collection_directory
|
||||
from wrolpi.config_import import ConfigSection, Unreadable, media_relative
|
||||
from wrolpi.dates import now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, session_curs
|
||||
from wrolpi.db import get_db_session, session_curs
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \
|
||||
|
|
@ -1357,50 +1357,6 @@ def get_domains():
|
|||
return domains
|
||||
|
||||
|
||||
def get_domain(domain_id: int) -> dict:
|
||||
"""
|
||||
Get a single domain collection by ID with its archive statistics.
|
||||
|
||||
Returns a dict with collection details including id, domain name, url_count, size,
|
||||
tag_name, directory (relative to media directory), and description.
|
||||
|
||||
Raises UnknownArchive if domain not found.
|
||||
"""
|
||||
# Get the collection using find_by_id
|
||||
with get_db_session() as session:
|
||||
try:
|
||||
collection = Collection.find_by_id(session, domain_id)
|
||||
except Exception:
|
||||
# Collection.find_by_id raises UnknownCollection, but we want UnknownArchive
|
||||
raise UnknownArchive(f"Domain collection with ID {domain_id} not found")
|
||||
|
||||
if collection.kind != 'domain':
|
||||
raise UnknownArchive(f"Collection {domain_id} is not a domain")
|
||||
|
||||
# Get base domain data from __json__()
|
||||
domain_data = collection.__json__()
|
||||
|
||||
# Get archive statistics for this domain
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT COUNT(a.id) AS url_count, SUM(fg.size) AS size
|
||||
FROM archive a
|
||||
LEFT JOIN file_group fg on fg.id = a.file_group_id
|
||||
WHERE a.collection_id = :domain_id
|
||||
'''
|
||||
curs.execute(stmt, {'domain_id': domain_id})
|
||||
stats = dict(curs.fetchone())
|
||||
|
||||
# Enhance with archive stats
|
||||
domain_data['url_count'] = stats['url_count'] or 0
|
||||
domain_data['size'] = stats['size'] or 0
|
||||
|
||||
# Use 'domain' key for domain collections instead of 'name'
|
||||
domain_data['domain'] = domain_data.pop('name')
|
||||
|
||||
return domain_data
|
||||
|
||||
|
||||
ARCHIVE_ORDERS = {
|
||||
# Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime)
|
||||
'published_datetime': (date := 'fg.effective_datetime ASC NULLS LAST'),
|
||||
|
|
|
|||
|
|
@ -204,7 +204,7 @@ def _auto_create_collections(session: Session, file_group: FileGroup, metadata):
|
|||
|
||||
|
||||
@register_refresh_cleanup
|
||||
def doc_cleanup(test_directory=None):
|
||||
def doc_cleanup():
|
||||
"""Remove empty author/subject collections after refresh."""
|
||||
from sqlalchemy import func
|
||||
from wrolpi.collections.models import Collection, CollectionItem
|
||||
|
|
|
|||
|
|
@ -26,12 +26,6 @@ DOC_MIMETYPES = (
|
|||
'application/x-cb7',
|
||||
)
|
||||
|
||||
# Mimetypes that are considered ebook formats.
|
||||
EBOOK_MIMETYPES = (
|
||||
EPUB_MIMETYPE,
|
||||
MOBI_MIMETYPE,
|
||||
)
|
||||
|
||||
|
||||
COMIC_BOOK_SUFFIXES = ('.cbz', '.cbr', '.cbt', '.cb7')
|
||||
|
||||
|
|
@ -135,11 +129,6 @@ class Doc(ModelHelper, Base):
|
|||
session.delete(fg)
|
||||
|
||||
|
||||
# Section kinds.
|
||||
SECTION_KIND_EPUB_SPINE = 'epub_spine'
|
||||
SECTION_KIND_PDF_PAGE = 'pdf_page'
|
||||
|
||||
|
||||
class DocSection(ModelHelper, Base):
|
||||
"""A searchable sub-range of a Doc.
|
||||
|
||||
|
|
|
|||
|
|
@ -22,23 +22,6 @@ class DocSearchResponse:
|
|||
totals: dict = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocStatistics:
|
||||
doc_count: int = 0
|
||||
epub_count: int = 0
|
||||
pdf_count: int = 0
|
||||
other_count: int = 0
|
||||
total_size: int = 0
|
||||
author_count: int = 0
|
||||
subject_count: int = 0
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocStatisticsResponse:
|
||||
statistics: dict = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DocFileFormatRequest:
|
||||
name: str = ''
|
||||
model_tag: str = 'docs'
|
||||
|
|
|
|||
|
|
@ -5,14 +5,12 @@ All files are loaded into one shared, cross-worker dict keyed by slug. Unit mat
|
|||
frontend (mathjs), so this module treats item field values as opaque strings/numbers.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from wrolpi.common import logger, MultiFileConfig, read_config_data
|
||||
from wrolpi.errors import ValidationError
|
||||
from .defaults import (DEFAULT_INVENTORIES, FIELD_TYPES, default_fields, slugify)
|
||||
|
||||
MY_DIR: Path = Path(__file__).parent
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -94,7 +94,6 @@ DEFAULT_FIELD_SETS = {
|
|||
# The first/default type used when a type is not specified (e.g. legacy migration).
|
||||
DEFAULT_TYPE = 'food'
|
||||
|
||||
INVENTORY_TYPES = tuple(DEFAULT_FIELD_SETS.keys())
|
||||
|
||||
# Recommended package counts for a one-adult, one-year (~3,000 kcal/day) food supply, keyed by the shipped
|
||||
# food-catalog id (see catalog_defaults.yaml). Transcribed from the same "One-Year Emergency Food Supply for One
|
||||
|
|
|
|||
|
|
@ -9,12 +9,6 @@ class UnknownInventory(APIError):
|
|||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class InvalidFieldSchema(APIError):
|
||||
code = 'INVALID_FIELD_SCHEMA'
|
||||
summary = 'The inventory field schema is invalid'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class InventoryConflict(APIError):
|
||||
code = 'INVENTORY_CONFLICT'
|
||||
summary = 'The inventory was modified since it was loaded'
|
||||
|
|
|
|||
|
|
@ -230,4 +230,3 @@ MAP_REGIONS = [
|
|||
]
|
||||
|
||||
MAP_REGIONS_BY_NAME = {r['name']: r for r in MAP_REGIONS}
|
||||
MAP_REGIONS_BY_REGION = {r['region']: r for r in MAP_REGIONS}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from typing import List, Dict
|
|||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.map.catalog import MAP_REGIONS, MAP_REGIONS_BY_NAME, MAP_REGIONS_BY_REGION, MANIFEST_URL
|
||||
from modules.map.catalog import MAP_REGIONS, MAP_REGIONS_BY_NAME, MANIFEST_URL
|
||||
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config, aiohttp_get, verify_gpg_signature
|
||||
from wrolpi.downloader import DownloadFrequency, Download, save_downloads_config, download_manager
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from wrolpi.collections import Collection
|
|||
from wrolpi.common import logger, \
|
||||
get_media_directory, wrol_mode_check, background_task
|
||||
from wrolpi.db import get_db_session, session_curs
|
||||
from wrolpi.downloader import save_downloads_config, download_manager, Download
|
||||
from wrolpi.errors import APIError, ValidationError, FileWorkerConflict
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.tags import Tag
|
||||
|
|
@ -242,40 +241,6 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5,
|
|||
return channels
|
||||
|
||||
|
||||
@wrol_mode_check
|
||||
async def create_channel_download(channel_id: int, url: str, frequency: int, settings: dict):
|
||||
"""Create Download for Channel."""
|
||||
with get_db_session(commit=True) as session:
|
||||
channel = Channel.find_by_id(session, channel_id)
|
||||
download = channel.get_or_create_download(session, url, frequency, reset_attempts=True)
|
||||
download.settings = settings
|
||||
|
||||
save_channels_config.activate_switch()
|
||||
save_downloads_config.activate_switch()
|
||||
|
||||
return download
|
||||
|
||||
|
||||
@wrol_mode_check
|
||||
async def update_channel_download(channel_id: int, download_id: int, url: str, frequency: int, settings: dict):
|
||||
"""Fetch Channel's Download, update its properties."""
|
||||
with get_db_session(commit=True) as session:
|
||||
# Ensure that the Channel exists.
|
||||
Channel.find_by_id(session, channel_id)
|
||||
download = Download.find_by_id(session, download_id)
|
||||
download.url = url
|
||||
download.frequency = frequency
|
||||
download.settings = settings
|
||||
session.commit()
|
||||
|
||||
download_manager.remove_from_skip_list(url)
|
||||
|
||||
save_channels_config.activate_switch()
|
||||
save_downloads_config.activate_switch()
|
||||
|
||||
return download
|
||||
|
||||
|
||||
async def _background_move_channel(channel_id: int, target_directory: pathlib.Path):
|
||||
"""Helper to run channel move in background with its own database session.
|
||||
|
||||
|
|
|
|||
|
|
@ -25,10 +25,6 @@ from .errors import ChannelNameConflict, ChannelURLConflict, ChannelDirectoryCon
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_videos_directory() -> pathlib.Path:
|
||||
"""Get the "videos" directory in the media directory. Make it if it does not exist."""
|
||||
from modules.videos.lib import format_videos_destination
|
||||
|
|
@ -116,13 +112,6 @@ def duration_to_seconds(duration: str) -> int:
|
|||
).total_seconds())
|
||||
|
||||
|
||||
def seconds_to_duration(seconds: int) -> str:
|
||||
"""Convert integer seconds to ffmpeg duration"""
|
||||
hours, seconds = divmod(seconds, 3600)
|
||||
minutes, seconds = divmod(seconds, 60)
|
||||
return f'{hours:02}:{minutes:02}:{seconds:02}.000'
|
||||
|
||||
|
||||
DURATION_REGEX = re.compile(r'Duration: ([\d:.]+),')
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,12 +15,6 @@ class UnknownChannel(APIError):
|
|||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class InvalidChannel(APIError):
|
||||
code = 'INVALID_CHANNEL'
|
||||
summary = 'The channel is not supported'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class ChannelNameConflict(APIError):
|
||||
code = 'CHANNEL_NAME_CONFLICT'
|
||||
summary = 'The channel name is already taken.'
|
||||
|
|
@ -43,9 +37,3 @@ class ChannelSourceIdConflict(APIError):
|
|||
code = 'CHANNEL_SOURCE_ID_CONFLICT'
|
||||
summary = 'Search is empty, search_str must have content.'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class ChannelURLEmpty(APIError):
|
||||
code = 'CHANNEL_URL_EMPTY'
|
||||
summary = 'This channel has no URL'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from wrolpi.collections import Collection
|
|||
from wrolpi.collections.models import validate_collection_directory
|
||||
from wrolpi.config_import import ConfigSection, Unreadable, chunks, media_relative
|
||||
from wrolpi.config_version import register_ownership
|
||||
from wrolpi.dates import Seconds, from_timestamp
|
||||
from wrolpi.dates import from_timestamp
|
||||
from wrolpi.db import get_db_curs, get_db_session, session_curs
|
||||
from wrolpi.downloader import Download, download_manager, save_downloads_config
|
||||
from wrolpi.errors import UnknownDirectory
|
||||
|
|
@ -32,7 +32,7 @@ from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
|
|||
from wrolpi.tags import Tag, unknown_tag_names
|
||||
from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
|
||||
from .common import is_valid_poster, convert_image, \
|
||||
generate_video_poster, ConfigError, \
|
||||
generate_video_poster, \
|
||||
extract_video_duration, ffprobe_json_sync, read_ffprobe_json_file, write_ffprobe_json_file
|
||||
from .cookies import cookies_unlocked, cookies_for_download
|
||||
from .errors import UnknownChannel
|
||||
|
|
@ -40,9 +40,6 @@ from .models import Channel
|
|||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week
|
||||
|
||||
REQUIRED_OPTIONS = ['name', 'directory']
|
||||
|
||||
@dataclasses.dataclass
|
||||
class VideoInfoJSON:
|
||||
|
|
|
|||
|
|
@ -480,10 +480,6 @@ class Video(ModelHelper, Base):
|
|||
streams = [i for i in self.ffprobe_json['streams'] if i['codec_type'] == codec_type]
|
||||
return streams
|
||||
|
||||
def detect_is_complete(self):
|
||||
from modules.videos.common import ffmpeg_video_complete
|
||||
return ffmpeg_video_complete(self.video_path)
|
||||
|
||||
def get_channel_entry(self) -> Optional[Dict]:
|
||||
"""Return the info_json entry for this Video from its Channel."""
|
||||
if self.channel and self.source_id:
|
||||
|
|
@ -806,69 +802,6 @@ class Channel(ModelHelper, Base):
|
|||
download.collection_id = self.collection_id
|
||||
session.add(download)
|
||||
|
||||
def batch_update(self, data: dict, tags_by_name: dict = None, existing_downloads: dict = None):
|
||||
"""
|
||||
Update the attributes of this Channel using pre-fetched lookup dictionaries.
|
||||
More efficient than update() when processing many channels because it avoids
|
||||
individual database queries for tags and downloads.
|
||||
|
||||
Args:
|
||||
data: Config data dict with channel fields
|
||||
tags_by_name: Pre-fetched {tag_name: Tag} lookup dictionary
|
||||
existing_downloads: Pre-fetched {url: Download} lookup dictionary
|
||||
"""
|
||||
data = data.copy()
|
||||
tags_by_name = tags_by_name or {}
|
||||
existing_downloads = existing_downloads or {}
|
||||
|
||||
# URL should not be empty string.
|
||||
url = data.pop('url', None)
|
||||
self.url = url or None
|
||||
|
||||
session: Session = Session.object_session(self)
|
||||
downloads = data.pop('downloads', [])
|
||||
|
||||
# Use pre-fetched tag lookup instead of querying
|
||||
if tag_name := data.pop('tag_name', None):
|
||||
tag = tags_by_name.get(tag_name)
|
||||
if tag:
|
||||
self.tag = tag
|
||||
else:
|
||||
logger.warning(f'Tag {repr(tag_name)} not found for channel')
|
||||
|
||||
for key, value in data.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
# We need an absolute directory.
|
||||
if isinstance(self.directory, pathlib.Path) and not self.directory.is_absolute():
|
||||
self.directory = get_media_directory() / self.directory
|
||||
elif isinstance(self.directory, str) and not pathlib.Path(self.directory).is_absolute():
|
||||
self.directory = get_media_directory() / self.directory
|
||||
|
||||
for download in downloads:
|
||||
url = None
|
||||
frequency = None
|
||||
if isinstance(download, dict):
|
||||
url = download['url']
|
||||
# Use pre-fetched download lookup instead of querying
|
||||
if existing_download := existing_downloads.get(url):
|
||||
frequency = existing_download.frequency
|
||||
else:
|
||||
frequency = download.get('frequency')
|
||||
elif isinstance(download, Download):
|
||||
url = download.url
|
||||
frequency = download.frequency
|
||||
|
||||
if not url:
|
||||
raise RuntimeError(f'Unknown download type: {download}')
|
||||
if not frequency:
|
||||
logger.error(f'Refusing to create Download for Channel without frequency: {url}')
|
||||
continue
|
||||
|
||||
download_obj = self.get_or_create_download(session, url, frequency, reset_attempts=True)
|
||||
download_obj.collection_id = self.collection_id
|
||||
session.add(download_obj)
|
||||
|
||||
def config_view(self) -> dict:
|
||||
"""
|
||||
Retrieve the data about this Channel that should be stored in a config file.
|
||||
|
|
|
|||
|
|
@ -55,27 +55,6 @@ class ChannelTagInfoRequest:
|
|||
tag_name: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelDownloadRequest:
|
||||
url: str
|
||||
frequency: int
|
||||
settings: dict = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self):
|
||||
self.url = self.url.strip()
|
||||
if not self.url:
|
||||
raise ValidationError('url cannot be empty')
|
||||
|
||||
# Validate settings contents. Remove empty values.
|
||||
from wrolpi.schema import DownloadSettings
|
||||
self.settings = {k: v for k, v in DownloadSettings(**self.settings).__dict__.items() if v not in ([], None)}
|
||||
|
||||
|
||||
@dataclass
|
||||
class RefreshRequest:
|
||||
channel_name: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelModel:
|
||||
id: int
|
||||
|
|
@ -105,12 +84,6 @@ class ChannelsResponse:
|
|||
channels: List[ChannelsChannelModel]
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamResponse:
|
||||
success: str
|
||||
stream_url: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Video:
|
||||
id: int
|
||||
|
|
@ -210,22 +183,12 @@ class VideoSearchResponse:
|
|||
tsquery: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class PaginationQuery:
|
||||
offset: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class TagRequest:
|
||||
video_id: int
|
||||
tag_name: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class WROLModeRequest:
|
||||
enabled: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoStatistics:
|
||||
videos: int
|
||||
|
|
@ -252,17 +215,6 @@ class VideosStatisticsResponse:
|
|||
channels = ChannelStatistics
|
||||
|
||||
|
||||
@dataclass
|
||||
class CensoredVideoRequest:
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class CensoredVideoResponse:
|
||||
videos: List[VideoWithChannel]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelSearchRequest:
|
||||
tag_names: List[str] = field(default_factory=list)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue