From 22ada7eb575d8883af13740fb271d7ec6fc937cc Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:50:09 -0600 Subject: [PATCH] Remove unused code found by vulture (#885) Deletes functions, methods, classes, schema dataclasses, error classes, Events senders, and module-level constants that nothing references, including tests. Also drops the unused `doc_cleanup(test_directory)` and `_wait_for_sub_job_with_progress(collection_idx)` parameters, and the imports that only served removed code. Co-authored-by: Claude Opus 5.5 --- controller/api/schemas.py | 27 ------------- modules/archive/__init__.py | 1 - modules/archive/lib.py | 46 +---------------------- modules/docs/__init__.py | 2 +- modules/docs/models.py | 11 ------ modules/docs/schema.py | 17 --------- modules/inventory/common.py | 2 - modules/inventory/defaults.py | 1 - modules/inventory/errors.py | 6 --- modules/map/catalog.py | 1 - modules/map/lib.py | 2 +- modules/videos/channel/lib.py | 35 ----------------- modules/videos/common.py | 11 ------ modules/videos/errors.py | 12 ------ modules/videos/lib.py | 7 +--- modules/videos/models.py | 67 --------------------------------- modules/videos/schema.py | 48 ----------------------- wrolpi/cmd.py | 10 ----- wrolpi/collections/models.py | 69 +--------------------------------- wrolpi/collections/schema.py | 21 ----------- wrolpi/common.py | 4 -- wrolpi/config_schema.py | 10 ----- wrolpi/dates.py | 5 --- wrolpi/db.py | 22 +---------- wrolpi/downloader.py | 8 ---- wrolpi/errors.py | 24 ------------ wrolpi/events.py | 20 ---------- wrolpi/files/models.py | 32 ---------------- wrolpi/files/refresh.py | 6 --- wrolpi/files/reorganize_job.py | 3 +- wrolpi/fts.py | 7 ---- wrolpi/schema.py | 36 ------------------ wrolpi/tags.py | 16 -------- wrolpi/vars.py | 6 --- 34 files changed, 9 insertions(+), 586 deletions(-) diff --git a/controller/api/schemas.py b/controller/api/schemas.py index 3d32a284..f5befe43 100644 --- a/controller/api/schemas.py +++ b/controller/api/schemas.py @@ -82,22 +82,6 @@ class DriveStatusResponse(BaseModel): free_gb: float = Field(description="Free space in GB") -class PrimaryDriveStatusResponse(BaseModel): - """Response model for /api/stats/drives/primary endpoint.""" - - mounted: bool = Field(description="Whether the primary drive is mounted") - device: Optional[str] = Field(default=None, description="Device path") - mount_point: Optional[str] = Field(default=None, description="Mount point path") - fstype: Optional[str] = Field(default=None, description="Filesystem type") - total_bytes: Optional[int] = Field(default=None, description="Total space in bytes") - used_bytes: Optional[int] = Field(default=None, description="Used space in bytes") - free_bytes: Optional[int] = Field(default=None, description="Free space in bytes") - percent: Optional[float] = Field(default=None, description="Usage percentage") - total_gb: Optional[float] = Field(default=None, description="Total space in GB") - used_gb: Optional[float] = Field(default=None, description="Used space in GB") - free_gb: Optional[float] = Field(default=None, description="Free space in GB") - - # ============================================================================ # Status - Network # ============================================================================ @@ -132,17 +116,6 @@ class PowerStatusResponse(BaseModel): # Status - Full # ============================================================================ -class FullStatusResponse(BaseModel): - """Response model for /api/stats endpoint.""" - - cpu: CpuStatusResponse = Field(description="CPU status") - memory: MemoryStatusResponse = Field(description="Memory status") - load: LoadStatusResponse = Field(description="System load") - drives: list[DriveStatusResponse] = Field(description="All mounted drives") - primary_drive: Optional[DriveStatusResponse] = Field(default=None, description="Primary WROLPi drive") - network: list[NetworkInterfaceResponse] = Field(description="Network interfaces") - power: PowerStatusResponse = Field(description="Power status") - # ============================================================================ # Admin - Hotspot diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 3f820560..21bde1e2 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -26,7 +26,6 @@ from .lib import is_singlefile_file, request_archive, SINGLEFILE_HEADER, get_url SinglefileArtifacts, WrittenArchiveFiles, extract_singlefile_artifacts, register_archive, write_archive_files from .models import Archive -PRETTY_NAME = 'Archive' logger = logger.getChild(__name__) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index ba90469e..500b5075 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -30,7 +30,7 @@ from wrolpi.common import get_media_directory, get_relative_to_media_directory, from wrolpi.collections.models import validate_collection_directory from wrolpi.config_import import ConfigSection, Unreadable, media_relative from wrolpi.dates import now, Seconds -from wrolpi.db import get_db_session, get_db_curs, session_curs +from wrolpi.db import get_db_session, session_curs from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime from wrolpi.events import Events from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \ @@ -1357,50 +1357,6 @@ def get_domains(): return domains -def get_domain(domain_id: int) -> dict: - """ - Get a single domain collection by ID with its archive statistics. - - Returns a dict with collection details including id, domain name, url_count, size, - tag_name, directory (relative to media directory), and description. - - Raises UnknownArchive if domain not found. - """ - # Get the collection using find_by_id - with get_db_session() as session: - try: - collection = Collection.find_by_id(session, domain_id) - except Exception: - # Collection.find_by_id raises UnknownCollection, but we want UnknownArchive - raise UnknownArchive(f"Domain collection with ID {domain_id} not found") - - if collection.kind != 'domain': - raise UnknownArchive(f"Collection {domain_id} is not a domain") - - # Get base domain data from __json__() - domain_data = collection.__json__() - - # Get archive statistics for this domain - with get_db_curs() as curs: - stmt = ''' - SELECT COUNT(a.id) AS url_count, SUM(fg.size) AS size - FROM archive a - LEFT JOIN file_group fg on fg.id = a.file_group_id - WHERE a.collection_id = :domain_id - ''' - curs.execute(stmt, {'domain_id': domain_id}) - stats = dict(curs.fetchone()) - - # Enhance with archive stats - domain_data['url_count'] = stats['url_count'] or 0 - domain_data['size'] = stats['size'] or 0 - - # Use 'domain' key for domain collections instead of 'name' - domain_data['domain'] = domain_data.pop('name') - - return domain_data - - ARCHIVE_ORDERS = { # Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime) 'published_datetime': (date := 'fg.effective_datetime ASC NULLS LAST'), diff --git a/modules/docs/__init__.py b/modules/docs/__init__.py index b7454156..efa9d9db 100644 --- a/modules/docs/__init__.py +++ b/modules/docs/__init__.py @@ -204,7 +204,7 @@ def _auto_create_collections(session: Session, file_group: FileGroup, metadata): @register_refresh_cleanup -def doc_cleanup(test_directory=None): +def doc_cleanup(): """Remove empty author/subject collections after refresh.""" from sqlalchemy import func from wrolpi.collections.models import Collection, CollectionItem diff --git a/modules/docs/models.py b/modules/docs/models.py index 793782fc..16f35fcf 100644 --- a/modules/docs/models.py +++ b/modules/docs/models.py @@ -26,12 +26,6 @@ DOC_MIMETYPES = ( 'application/x-cb7', ) -# Mimetypes that are considered ebook formats. -EBOOK_MIMETYPES = ( - EPUB_MIMETYPE, - MOBI_MIMETYPE, -) - COMIC_BOOK_SUFFIXES = ('.cbz', '.cbr', '.cbt', '.cb7') @@ -135,11 +129,6 @@ class Doc(ModelHelper, Base): session.delete(fg) -# Section kinds. -SECTION_KIND_EPUB_SPINE = 'epub_spine' -SECTION_KIND_PDF_PAGE = 'pdf_page' - - class DocSection(ModelHelper, Base): """A searchable sub-range of a Doc. diff --git a/modules/docs/schema.py b/modules/docs/schema.py index c95a09d9..9c3ad029 100644 --- a/modules/docs/schema.py +++ b/modules/docs/schema.py @@ -22,23 +22,6 @@ class DocSearchResponse: totals: dict = dataclasses.field(default_factory=dict) -@dataclasses.dataclass -class DocStatistics: - doc_count: int = 0 - epub_count: int = 0 - pdf_count: int = 0 - other_count: int = 0 - total_size: int = 0 - author_count: int = 0 - subject_count: int = 0 - - @dataclasses.dataclass class DocStatisticsResponse: statistics: dict = dataclasses.field(default_factory=dict) - - -@dataclasses.dataclass -class DocFileFormatRequest: - name: str = '' - model_tag: str = 'docs' diff --git a/modules/inventory/common.py b/modules/inventory/common.py index d65333d5..1acf5025 100644 --- a/modules/inventory/common.py +++ b/modules/inventory/common.py @@ -5,14 +5,12 @@ All files are loaded into one shared, cross-worker dict keyed by slug. Unit mat frontend (mathjs), so this module treats item field values as opaque strings/numbers. """ from dataclasses import dataclass, field -from pathlib import Path from typing import List, Optional from wrolpi.common import logger, MultiFileConfig, read_config_data from wrolpi.errors import ValidationError from .defaults import (DEFAULT_INVENTORIES, FIELD_TYPES, default_fields, slugify) -MY_DIR: Path = Path(__file__).parent logger = logger.getChild(__name__) diff --git a/modules/inventory/defaults.py b/modules/inventory/defaults.py index d99b1420..3e6de5e2 100644 --- a/modules/inventory/defaults.py +++ b/modules/inventory/defaults.py @@ -94,7 +94,6 @@ DEFAULT_FIELD_SETS = { # The first/default type used when a type is not specified (e.g. legacy migration). DEFAULT_TYPE = 'food' -INVENTORY_TYPES = tuple(DEFAULT_FIELD_SETS.keys()) # Recommended package counts for a one-adult, one-year (~3,000 kcal/day) food supply, keyed by the shipped # food-catalog id (see catalog_defaults.yaml). Transcribed from the same "One-Year Emergency Food Supply for One diff --git a/modules/inventory/errors.py b/modules/inventory/errors.py index f41cc7c8..100cdc14 100644 --- a/modules/inventory/errors.py +++ b/modules/inventory/errors.py @@ -9,12 +9,6 @@ class UnknownInventory(APIError): status_code = HTTPStatus.NOT_FOUND -class InvalidFieldSchema(APIError): - code = 'INVALID_FIELD_SCHEMA' - summary = 'The inventory field schema is invalid' - status_code = HTTPStatus.BAD_REQUEST - - class InventoryConflict(APIError): code = 'INVENTORY_CONFLICT' summary = 'The inventory was modified since it was loaded' diff --git a/modules/map/catalog.py b/modules/map/catalog.py index a12a1189..2ccda88d 100644 --- a/modules/map/catalog.py +++ b/modules/map/catalog.py @@ -230,4 +230,3 @@ MAP_REGIONS = [ ] MAP_REGIONS_BY_NAME = {r['name']: r for r in MAP_REGIONS} -MAP_REGIONS_BY_REGION = {r['region']: r for r in MAP_REGIONS} diff --git a/modules/map/lib.py b/modules/map/lib.py index 5e706b41..84526e8a 100644 --- a/modules/map/lib.py +++ b/modules/map/lib.py @@ -5,7 +5,7 @@ from typing import List, Dict from sqlalchemy.orm import Session -from modules.map.catalog import MAP_REGIONS, MAP_REGIONS_BY_NAME, MAP_REGIONS_BY_REGION, MANIFEST_URL +from modules.map.catalog import MAP_REGIONS, MAP_REGIONS_BY_NAME, MANIFEST_URL from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config, aiohttp_get, verify_gpg_signature from wrolpi.downloader import DownloadFrequency, Download, save_downloads_config, download_manager diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index dad68f6a..ca2937e4 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -10,7 +10,6 @@ from wrolpi.collections import Collection from wrolpi.common import logger, \ get_media_directory, wrol_mode_check, background_task from wrolpi.db import get_db_session, session_curs -from wrolpi.downloader import save_downloads_config, download_manager, Download from wrolpi.errors import APIError, ValidationError, FileWorkerConflict from wrolpi.events import Events from wrolpi.tags import Tag @@ -242,40 +241,6 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5, return channels -@wrol_mode_check -async def create_channel_download(channel_id: int, url: str, frequency: int, settings: dict): - """Create Download for Channel.""" - with get_db_session(commit=True) as session: - channel = Channel.find_by_id(session, channel_id) - download = channel.get_or_create_download(session, url, frequency, reset_attempts=True) - download.settings = settings - - save_channels_config.activate_switch() - save_downloads_config.activate_switch() - - return download - - -@wrol_mode_check -async def update_channel_download(channel_id: int, download_id: int, url: str, frequency: int, settings: dict): - """Fetch Channel's Download, update its properties.""" - with get_db_session(commit=True) as session: - # Ensure that the Channel exists. - Channel.find_by_id(session, channel_id) - download = Download.find_by_id(session, download_id) - download.url = url - download.frequency = frequency - download.settings = settings - session.commit() - - download_manager.remove_from_skip_list(url) - - save_channels_config.activate_switch() - save_downloads_config.activate_switch() - - return download - - async def _background_move_channel(channel_id: int, target_directory: pathlib.Path): """Helper to run channel move in background with its own database session. diff --git a/modules/videos/common.py b/modules/videos/common.py index 6ae98dc9..723ef9cb 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -25,10 +25,6 @@ from .errors import ChannelNameConflict, ChannelURLConflict, ChannelDirectoryCon logger = logger.getChild(__name__) -class ConfigError(Exception): - pass - - def get_videos_directory() -> pathlib.Path: """Get the "videos" directory in the media directory. Make it if it does not exist.""" from modules.videos.lib import format_videos_destination @@ -116,13 +112,6 @@ def duration_to_seconds(duration: str) -> int: ).total_seconds()) -def seconds_to_duration(seconds: int) -> str: - """Convert integer seconds to ffmpeg duration""" - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return f'{hours:02}:{minutes:02}:{seconds:02}.000' - - DURATION_REGEX = re.compile(r'Duration: ([\d:.]+),') diff --git a/modules/videos/errors.py b/modules/videos/errors.py index 258fdfe0..93cc7bf4 100644 --- a/modules/videos/errors.py +++ b/modules/videos/errors.py @@ -15,12 +15,6 @@ class UnknownChannel(APIError): status_code = HTTPStatus.NOT_FOUND -class InvalidChannel(APIError): - code = 'INVALID_CHANNEL' - summary = 'The channel is not supported' - status_code = HTTPStatus.BAD_REQUEST - - class ChannelNameConflict(APIError): code = 'CHANNEL_NAME_CONFLICT' summary = 'The channel name is already taken.' @@ -43,9 +37,3 @@ class ChannelSourceIdConflict(APIError): code = 'CHANNEL_SOURCE_ID_CONFLICT' summary = 'Search is empty, search_str must have content.' status_code = HTTPStatus.BAD_REQUEST - - -class ChannelURLEmpty(APIError): - code = 'CHANNEL_URL_EMPTY' - summary = 'This channel has no URL' - status_code = HTTPStatus.BAD_REQUEST diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 07a995fc..6ad61e56 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -22,7 +22,7 @@ from wrolpi.collections import Collection from wrolpi.collections.models import validate_collection_directory from wrolpi.config_import import ConfigSection, Unreadable, chunks, media_relative from wrolpi.config_version import register_ownership -from wrolpi.dates import Seconds, from_timestamp +from wrolpi.dates import from_timestamp from wrolpi.db import get_db_curs, get_db_session, session_curs from wrolpi.downloader import Download, download_manager, save_downloads_config from wrolpi.errors import UnknownDirectory @@ -32,7 +32,7 @@ from wrolpi.switches import register_switch_handler, ActivateSwitchMethod from wrolpi.tags import Tag, unknown_tag_names from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST from .common import is_valid_poster, convert_image, \ - generate_video_poster, ConfigError, \ + generate_video_poster, \ extract_video_duration, ffprobe_json_sync, read_ffprobe_json_file, write_ffprobe_json_file from .cookies import cookies_unlocked, cookies_for_download from .errors import UnknownChannel @@ -40,9 +40,6 @@ from .models import Channel logger = logger.getChild(__name__) -DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week - -REQUIRED_OPTIONS = ['name', 'directory'] @dataclasses.dataclass class VideoInfoJSON: diff --git a/modules/videos/models.py b/modules/videos/models.py index 9121e594..dc44f4a1 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -480,10 +480,6 @@ class Video(ModelHelper, Base): streams = [i for i in self.ffprobe_json['streams'] if i['codec_type'] == codec_type] return streams - def detect_is_complete(self): - from modules.videos.common import ffmpeg_video_complete - return ffmpeg_video_complete(self.video_path) - def get_channel_entry(self) -> Optional[Dict]: """Return the info_json entry for this Video from its Channel.""" if self.channel and self.source_id: @@ -806,69 +802,6 @@ class Channel(ModelHelper, Base): download.collection_id = self.collection_id session.add(download) - def batch_update(self, data: dict, tags_by_name: dict = None, existing_downloads: dict = None): - """ - Update the attributes of this Channel using pre-fetched lookup dictionaries. - More efficient than update() when processing many channels because it avoids - individual database queries for tags and downloads. - - Args: - data: Config data dict with channel fields - tags_by_name: Pre-fetched {tag_name: Tag} lookup dictionary - existing_downloads: Pre-fetched {url: Download} lookup dictionary - """ - data = data.copy() - tags_by_name = tags_by_name or {} - existing_downloads = existing_downloads or {} - - # URL should not be empty string. - url = data.pop('url', None) - self.url = url or None - - session: Session = Session.object_session(self) - downloads = data.pop('downloads', []) - - # Use pre-fetched tag lookup instead of querying - if tag_name := data.pop('tag_name', None): - tag = tags_by_name.get(tag_name) - if tag: - self.tag = tag - else: - logger.warning(f'Tag {repr(tag_name)} not found for channel') - - for key, value in data.items(): - setattr(self, key, value) - - # We need an absolute directory. - if isinstance(self.directory, pathlib.Path) and not self.directory.is_absolute(): - self.directory = get_media_directory() / self.directory - elif isinstance(self.directory, str) and not pathlib.Path(self.directory).is_absolute(): - self.directory = get_media_directory() / self.directory - - for download in downloads: - url = None - frequency = None - if isinstance(download, dict): - url = download['url'] - # Use pre-fetched download lookup instead of querying - if existing_download := existing_downloads.get(url): - frequency = existing_download.frequency - else: - frequency = download.get('frequency') - elif isinstance(download, Download): - url = download.url - frequency = download.frequency - - if not url: - raise RuntimeError(f'Unknown download type: {download}') - if not frequency: - logger.error(f'Refusing to create Download for Channel without frequency: {url}') - continue - - download_obj = self.get_or_create_download(session, url, frequency, reset_attempts=True) - download_obj.collection_id = self.collection_id - session.add(download_obj) - def config_view(self) -> dict: """ Retrieve the data about this Channel that should be stored in a config file. diff --git a/modules/videos/schema.py b/modules/videos/schema.py index 0845b7db..9f7b294f 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -55,27 +55,6 @@ class ChannelTagInfoRequest: tag_name: Optional[str] = None -@dataclass -class ChannelDownloadRequest: - url: str - frequency: int - settings: dict = field(default_factory=dict) - - def __post_init__(self): - self.url = self.url.strip() - if not self.url: - raise ValidationError('url cannot be empty') - - # Validate settings contents. Remove empty values. - from wrolpi.schema import DownloadSettings - self.settings = {k: v for k, v in DownloadSettings(**self.settings).__dict__.items() if v not in ([], None)} - - -@dataclass -class RefreshRequest: - channel_name: str - - @dataclass class ChannelModel: id: int @@ -105,12 +84,6 @@ class ChannelsResponse: channels: List[ChannelsChannelModel] -@dataclass -class StreamResponse: - success: str - stream_url: str - - @dataclass class Video: id: int @@ -210,22 +183,12 @@ class VideoSearchResponse: tsquery: str -@dataclass -class PaginationQuery: - offset: int - - @dataclass class TagRequest: video_id: int tag_name: str -@dataclass -class WROLModeRequest: - enabled: bool - - @dataclass class VideoStatistics: videos: int @@ -252,17 +215,6 @@ class VideosStatisticsResponse: channels = ChannelStatistics -@dataclass -class CensoredVideoRequest: - limit: int - offset: int - - -@dataclass -class CensoredVideoResponse: - videos: List[VideoWithChannel] - - @dataclass class ChannelSearchRequest: tag_names: List[str] = field(default_factory=list) diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py index 13c313f2..cd41365d 100644 --- a/wrolpi/cmd.py +++ b/wrolpi/cmd.py @@ -60,18 +60,8 @@ def pid_is_running(pid: int) -> bool: return False -# Admin -SUDO_BIN = which('sudo', '/usr/bin/sudo') -NMCLI_BIN = which('nmcli', '/usr/bin/nmcli') -CPUFREQ_INFO_BIN = which('cpufreq-info', '/usr/bin/cpufreq-info') -CPUFREQ_SET_BIN = which('cpufreq-set', '/usr/bin/cpufreq-set') - -# Files -WGET_BIN = which('wget', '/usr/bin/wget') GIT_BIN = which('git', '/usr/bin/git') -# Map -BASH_BIN = which('bash', '/bin/bash') # Archive SINGLE_FILE_BIN = which('single-file', diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index d43ed432..b6e403df 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -12,7 +12,7 @@ from sqlalchemy.orm import Mapped, relationship, Session, joinedload, selectinlo from wrolpi import flags from wrolpi.common import loaded, Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \ unique_by_predicate, TRACE_LEVEL -from wrolpi.config_version import collection_kinds, count_config_change, register_ownership +from wrolpi.config_version import collection_kinds, register_ownership from wrolpi.db import serializer from wrolpi.downloader import Download, save_downloads_config from wrolpi.errors import ValidationError @@ -718,73 +718,6 @@ class Collection(ModelHelper, Base): # Shift remaining items down self._shift_positions(session, position + 1, shift_by=-1) - def remove_file_groups(self, session: Session, file_group_ids: List[int]): - """ - Remove multiple FileGroups from this Collection in batch. - More efficient than calling remove_file_group in a loop. - - Args: - session: Database session - file_group_ids: List of FileGroup IDs to remove - """ - - if not file_group_ids: - return - - # Delete all items at once - deleted = session.query(CollectionItem).filter( - CollectionItem.collection_id == self.id, - CollectionItem.file_group_id.in_(file_group_ids) - ).delete(synchronize_session=False) - - logger.debug(f'Removed {deleted} FileGroups from Collection {self.id}') - if deleted and self.kind == 'playlist': - # A bulk delete is not flushed: count it. - count_config_change(session, 'playlists.yaml') - - # Resequence positions to close gaps (optional, but keeps positions clean) - items = session.query(CollectionItem).filter_by( - collection_id=self.id - ).order_by(CollectionItem.position).all() - - for idx, item in enumerate(items, start=1): - item.position = idx - - session.flush(items) - - def reorder_item(self, file_group_id: int, new_position: int, session: Session = None): - """Move an item to a new position in the collection.""" - session = session or Session.object_session(self) - - item = session.query(CollectionItem).filter_by( - collection_id=self.id, - file_group_id=file_group_id - ).first() - - if not item: - raise ValueError(f'FileGroup {file_group_id} not in Collection {self.id}') - - old_position = item.position - - if old_position == new_position: - return - - # Remove item from old position - session.query(CollectionItem).filter( - CollectionItem.collection_id == self.id, - CollectionItem.position > old_position - ).update({'position': CollectionItem.position - 1}) - - # Make space at new position - session.query(CollectionItem).filter( - CollectionItem.collection_id == self.id, - CollectionItem.position >= new_position - ).update({'position': CollectionItem.position + 1}) - - # Update item position - item.position = new_position - session.flush() - def _shift_positions(self, session: Session, from_position: int, shift_by: int): """Helper to shift positions of items.""" session.query(CollectionItem).filter( diff --git a/wrolpi/collections/schema.py b/wrolpi/collections/schema.py index 55fa35d3..0deacc9f 100644 --- a/wrolpi/collections/schema.py +++ b/wrolpi/collections/schema.py @@ -147,16 +147,6 @@ class ReorganizationStatusResponse: # ============================================================================ -@dataclass -class BatchCollectionInfo: - """Info about a single collection in batch reorganization.""" - collection_id: int - collection_name: str - total_files: int - files_needing_move: int - sample_move: Optional[dict] = None - - @dataclass class BatchReorganizationListResponse: """Response for listing collections needing batch reorganization.""" @@ -174,17 +164,6 @@ class BatchReorganizationExecuteResponse: collection_count: int -@dataclass -class BatchCollectionProgress: - """Progress info for a single collection in batch reorganization.""" - id: int - name: str - status: str - total: int = 0 - completed: int = 0 - percent: int = 0 - - @dataclass class BatchReorganizationStatusResponse: """Response for batch reorganization status.""" diff --git a/wrolpi/common.py b/wrolpi/common.py index 62e23d61..48e99e98 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -12,7 +12,6 @@ import pathlib import re import shutil import socket -import string import sys import tempfile from asyncio import Task @@ -408,9 +407,6 @@ class ModelHelper: raise RuntimeError(f'{self} is not in a session!') -URL_CHARS = string.ascii_lowercase + string.digits - - def find_file(directory: pathlib.Path, name: str, depth=1) -> Optional[pathlib.Path]: """Recursively searches a directory for a file with the provided name.""" diff --git a/wrolpi/config_schema.py b/wrolpi/config_schema.py index f23a7644..56f67470 100644 --- a/wrolpi/config_schema.py +++ b/wrolpi/config_schema.py @@ -1,16 +1,6 @@ from dataclasses import dataclass -@dataclass -class ConfigGetRequest: - file_name: str = None - - -@dataclass -class ConfigPostRequest: - file_name: str - - @dataclass class ConfigsRequest: file_name: str diff --git a/wrolpi/dates.py b/wrolpi/dates.py index aedf3162..b6401223 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -7,7 +7,6 @@ import pytz from sqlalchemy import types from wrolpi.errors import InvalidDatetime -from wrolpi.vars import DATETIME_FORMAT_MS logger = logging.getLogger(__name__) @@ -121,10 +120,6 @@ def strpdate(dt: str) -> datetime: raise InvalidDatetime(f'Unable to parse datetime string: {dt}') from e -def strftime_ms(dt: datetime) -> str: - return dt.strftime(DATETIME_FORMAT_MS) - - def strptime_ms(dt: str) -> datetime: return datetime.fromisoformat(dt).astimezone(pytz.UTC) diff --git a/wrolpi/db.py b/wrolpi/db.py index f5fca63f..69ba0c37 100644 --- a/wrolpi/db.py +++ b/wrolpi/db.py @@ -15,7 +15,7 @@ import threading import traceback import types from contextlib import contextmanager -from typing import Tuple, List, Union, Type, Generator, Any, ContextManager +from typing import Tuple, List, Union, Generator, Any, ContextManager import sqlalchemy import sqlalchemy.exc @@ -23,7 +23,7 @@ from sqlalchemy import event from sqlalchemy.orm import sessionmaker, Session from sqlalchemy.pool import NullPool -from wrolpi.common import logger, Base +from wrolpi.common import logger from wrolpi.vars import PYTEST logger = logger.getChild(__name__) @@ -525,15 +525,6 @@ class RequestDB: _in_request.set(False) -def get_ranked_models(ranked_primary_keys: List, model: Type[Base], session: Session) -> List[Base]: - """Get all objects whose primary keys are in the `ranked_primary_keys`, preserve their order.""" - pkey = sqlalchemy.inspect(model).primary_key[0] - pkey_name = pkey.name - results = list(session.query(model).filter(pkey.in_(ranked_primary_keys)).all()) - results = sorted(results, key=lambda i: ranked_primary_keys.index(getattr(i, pkey_name))) - return results - - # SQLite's default variable limit; multi-row VALUES clauses must stay under it. SQLITE_MAX_VARIABLES = 32_000 @@ -570,13 +561,6 @@ def named_placeholders(prefix: str, values: List, params: dict) -> str: return ', '.join(names) -def json_each_in(param_name: str) -> str: - """An IN-clause subquery over a JSON array parameter; bind `json.dumps(list)` as the parameter. - - Immune to SQLite's variable-count limit regardless of list size.""" - return f'(SELECT value FROM json_each(:{param_name}))' - - def parse_db_datetime(value: Union[str, None]): """Parse a datetime TEXT value read by a raw cursor into a UTC-aware datetime. @@ -590,5 +574,3 @@ def parse_db_datetime(value: Union[str, None]): except ValueError: parsed = _datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') return parsed.replace(tzinfo=_datetime.timezone.utc) - - diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index fdad945a..a89c61ee 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -339,10 +339,6 @@ class Download(ModelHelper, Base): # noqa self.attempts += 1 self.status = DownloadStatus.pending - @property - def is_pending(self) -> bool: - return self.status == DownloadStatus.pending - def complete(self): """Mark this Download as successfully downloaded.""" self.status = DownloadStatus.complete @@ -1499,10 +1495,6 @@ class DownloadManager: [i for i in download_manager_data['killed_downloads'] if i != download_id] api_app.shared_ctx.download_manager_data.update(download_manager_data) - @staticmethod - def download_is_killed(download_id: int): - return download_id in api_app.shared_ctx.download_manager_data['killed_downloads'] - FINISHED_STATUSES = (DownloadStatus.complete, DownloadStatus.failed) def delete_old_once_downloads(self): diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 8485b454..3f2a0062 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -81,12 +81,6 @@ class NativeOnly(APIError): status_code = HTTPStatus.BAD_REQUEST -class HotspotError(APIError): - code = 'HOTSPOT_ERROR' - summary = 'Updating/accessing Hotspot encountered an error' - status_code = HTTPStatus.INTERNAL_SERVER_ERROR - - class LogLevelError(APIError): code = 'LOG_LEVEL_ERROR' summary = 'Log level is invalid' @@ -123,12 +117,6 @@ class FileGroupIsTagged(APIError): status_code = HTTPStatus.CONFLICT -class FileGroupAlreadyTagged(APIError): - code = 'FILE_GROUP_ALREADY_TAGGED' - summary = 'FileGroup is already tagged with this Tag' - status_code = HTTPStatus.CONFLICT - - class FileUploadFailed(APIError): code = 'FILE_UPLOAD_FAILED' summary = 'Failed to upload file' @@ -141,18 +129,6 @@ class FileConflict(APIError): status_code = HTTPStatus.CONFLICT -class HotspotPasswordTooShort(APIError): - code = 'HOTSPOT_PASSWORD_TOO_SHORT' - summary = 'Hotspot password must be at least 8 characters' - status_code = HTTPStatus.BAD_REQUEST - - -class ShutdownFailed(APIError): - code = 'SHUTDOWN_FAILED' - summary = 'Unable to shutdown the WROLPi' - status_code = HTTPStatus.INTERNAL_SERVER_ERROR - - class InvalidConfig(APIError): code = 'INVALID_CONFIG' summary = 'Config is invalid' diff --git a/wrolpi/events.py b/wrolpi/events.py index b53d8507..5a88bcd5 100644 --- a/wrolpi/events.py +++ b/wrolpi/events.py @@ -16,10 +16,6 @@ class Events: def send_global_refresh_started(message: str = None): send_event('global_refresh_started', message, subject='refresh') - @staticmethod - def send_refresh_completed(message: str = None): - send_event('refresh_completed', message, subject='refresh') - @staticmethod def send_global_refresh_modeling_completed(message: str = None): send_event('global_refresh_modeling_completed', message, subject='refresh') @@ -64,14 +60,6 @@ class Events: def send_created(message: str = None): send_event('created', message, subject='created') - @staticmethod - def send_map_import_complete(message: str = None): - send_event('map_import_complete', message, subject='map') - - @staticmethod - def send_map_import_failed(message: str = None): - send_event('map_import_failed', message, subject='map') - @staticmethod def send_map_search_complete(message: str = None): send_event('map_search_complete', message, subject='map') @@ -84,14 +72,6 @@ class Events: def send_tippecanoe_missing(message: str = None): send_event('tippecanoe_missing', message, subject='map') - @staticmethod - def send_shutdown(message: str = None): - send_event('shutdown', message, subject='shutdown') - - @staticmethod - def send_shutdown_failed(message: str = None): - send_event('shutdown_failed', message, subject='shutdown') - @staticmethod def send_file_move_completed(message: str = None): send_event('file_move_completed', message, subject='refresh') diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index e8482cd3..fb2cd94f 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -198,18 +198,6 @@ class FileGroup(ModelHelper, Base): return self.title return self.primary_path.name - @property - def full_primary_path(self) -> pathlib.Path: - """Returns the complete absolute path to the primary file. - - Handles both old format (primary_path is absolute) and new format (directory + relative primary_path). - """ - if self.directory: - # New format: directory + relative filename - return self.directory / self.primary_path - # Old format: primary_path is already absolute - return self.primary_path - def resolve_path(self, relative_path: str | pathlib.Path) -> pathlib.Path: """Convert a relative filename to an absolute path using this FileGroup's directory. @@ -370,12 +358,6 @@ class FileGroup(ModelHelper, Base): subtitle_paths = [i['path'] for i in self.my_subtitle_files()] return [i for i in text_files if i['path'] not in subtitle_paths] - def my_epub_files(self) -> List[dict]: - return self.my_files('application/epub') - - def my_mobi_files(self) -> List[dict]: - return self.my_files('application/x-mobipocket-ebook') - def my_html_files(self) -> List[dict]: html_files = self.my_files('text/html', 'application/octet-stream') html_files = [i for i in html_files if i['path'].name.endswith('.html')] @@ -638,20 +620,6 @@ class FileGroup(ModelHelper, Base): raise UnknownFile(f'Unable to find FileGroup with path {path}') return file_group - @classmethod - def find_by_directory(cls, session: Session, directory: pathlib.Path, recursive: bool = False) -> List['FileGroup']: - """Find all FileGroups in a directory (optionally recursive). - - Uses the indexed `directory` column for efficient lookups. - """ - from sqlalchemy import or_ - directory_str = str(directory) - if recursive: - return session.query(cls).filter( - or_(cls.directory == directory_str, cls.directory.like(f'{directory_str}/%')) - ).all() - return session.query(cls).filter(cls.directory == directory_str).all() - @property def tag_names(self) -> List[str]: result = [] diff --git a/wrolpi/files/refresh.py b/wrolpi/files/refresh.py index b58756d2..109640fa 100644 --- a/wrolpi/files/refresh.py +++ b/wrolpi/files/refresh.py @@ -377,12 +377,6 @@ def _find_args_for_roots(roots: list[Path]) -> list[str]: return find_args -async def _stream_filesystem_paths(root: Path | list[Path]) -> AsyncGenerator[str, None]: - """Stream file paths using find. Kept for tests that patch this generator.""" - async for directory, filename, stem, mtime in _stream_filesystem_entries(root): - yield str(Path(directory) / filename) - - def _split_nul_field_triples(buf: bytes) -> Tuple[list, bytes]: """Pull complete NUL-separated (directory, filename, mtime) triples out of `buf`. diff --git a/wrolpi/files/reorganize_job.py b/wrolpi/files/reorganize_job.py index 7101e3c6..3941fc43 100644 --- a/wrolpi/files/reorganize_job.py +++ b/wrolpi/files/reorganize_job.py @@ -411,7 +411,7 @@ class FileReorganizeMixin: # Wait for the reorganization job to complete # Poll for completion while updating batch status await self._wait_for_sub_job_with_progress( - job_id, batch_status, collection_id, collection_name, idx, total_collections + job_id, batch_status, collection_id, collection_name, total_collections ) # Mark this collection as completed @@ -488,7 +488,6 @@ class FileReorganizeMixin: batch_status: dict, collection_id: int, collection_name: str, - collection_idx: int, total_collections: int, ): """Wait for a sub-job to complete while updating batch progress. diff --git a/wrolpi/fts.py b/wrolpi/fts.py index cb7b2435..76f17910 100644 --- a/wrolpi/fts.py +++ b/wrolpi/fts.py @@ -24,7 +24,6 @@ from typing import List, Optional, Tuple from wrolpi.schema_ddl import execute_sql -FILE_GROUP_FTS_COLUMNS = ('a_text', 'b_text', 'c_text', 'd_text') # The fast (non-deep) path searches these columns only. ABC_COLUMNS = ('a_text', 'b_text', 'c_text') @@ -340,12 +339,6 @@ def rebuild_fts(conn): execute_sql(conn, "INSERT INTO doc_section_fts(doc_section_fts) VALUES('rebuild')") -def optimize_fts(curs): - """Merge FTS5 b-trees for faster queries; call after large refreshes.""" - curs.execute("INSERT INTO file_group_fts(file_group_fts) VALUES('optimize')") - curs.execute("INSERT INTO doc_section_fts(doc_section_fts) VALUES('optimize')") - - def fts_integrity_ok(curs) -> bool: """Verify the FTS5 indexes match their content tables.""" try: diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 869efd0a..a8019cb0 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -7,11 +7,6 @@ from wrolpi.common import get_relative_to_media_directory, unique_by_predicate from wrolpi.errors import InvalidDownload, ValidationError -@dataclass -class SettingsObject: - media_directory: str - - @dataclass class SettingsResponse: download_manager_disabled: bool @@ -178,32 +173,6 @@ class MemoryStatusStat: cached: int -@dataclass -class NicBandwidthStatusStat: - name: str - now: float - bytes_recv: int - bytes_sent: int - speed: int - bytes_recv_ps: int - bytes_sent_ps: int - elapsed: int - - -@dataclass -class DiskBandwidthStatusStat: - bytes_read: int - bytes_read_ps: int - bytes_write: int - bytes_write_ps: int - elapsed: int - max_read_ps: int - max_write_ps: int - name: str - now: float - speed: int - - @dataclass class DownloadsSummaryResponse: """`DownloadManager.get_summary()`, which /api/status returns as `downloads`. @@ -432,11 +401,6 @@ class TagRequest: color: str -@dataclass -class DeleteTagRequest: - name: str - - @dataclass class NotifyRequest: message: str diff --git a/wrolpi/tags.py b/wrolpi/tags.py index dfd079d2..a858e1f5 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -53,12 +53,6 @@ class TagFile(ModelHelper, Base): def get_by_primary_keys(session: Session, file_group_id: int, tag_id: int) -> Optional['TagFile']: return session.query(TagFile).filter_by(tag_id=tag_id, file_group_id=file_group_id).one_or_none() - @staticmethod - def get_by_tag_name(session: Session, file_group_id: int, tag_name: str) -> Optional['TagFile']: - return session.query(TagFile).join(Tag) \ - .filter(TagFile.file_group_id == file_group_id, Tag.name == tag_name) \ - .one_or_none() - # Tag.get_id_by_name def get_id_by_name_key(klass: Base, session: Session, name: str): @@ -110,16 +104,6 @@ class Tag(ModelHelper, Base): return tag.id raise UnknownTag(f'No Tag with name={name}') - @classmethod - @cachetools.cached(cache=get_name_by_id_cache, key=get_name_by_id_key) - def get_name_by_id(cls, session: Session, id_: int) -> str: - """Returns a Tag's name, if a Tag matches the provided ID. - - @raise UnknownTag: If no Tag matches the provided ID.""" - if tag := session.query(Tag).filter_by(id=id_).one_or_none(): - return tag.id - raise UnknownTag(f'No Tag with id={id_}') - @classmethod def invalidate_cache(cls): get_id_by_name_cache.clear() diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 5c9b8416..c2ae7f2a 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -67,10 +67,7 @@ REPOS_DIRECTORY = 'repos' MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY)) CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config' -MODULES_DIR: Path = PROJECT_DIR / 'modules' -DATE_FORMAT = '%Y-%M-%d' -DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' DATETIME_FORMAT_MS = '%Y-%m-%d %H:%M:%S.%f' DEFAULT_CPU_FREQUENCY = 'ondemand' @@ -83,7 +80,6 @@ API_AUTO_RELOAD = truthy_arg(os.environ.get('API_AUTO_RELOAD', DOCKERIZED)) API_ACCESS_LOG = truthy_arg(os.environ.get('API_ACCESS_LOG', DOCKERIZED)) API_DEBUG = truthy_arg(os.environ.get('API_DEBUG', False)) -FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100)) FILE_MAX_PDF_SIZE = int(os.environ.get('FILE_MAX_PDF_SIZE', 40_000_000)) FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000)) @@ -108,5 +104,3 @@ IS_MACOS = os.uname()[0] == 'Darwin' SIMULTANEOUS_DOWNLOAD_DOMAINS = int(os.environ.get('SIMULTANEOUS_DOWNLOAD_DOMAINS', 2 if IS_RPI else 4)) - -WROLPI_HOME = pathlib.Path(os.environ.get('WROLPI_HOME') or '/home/wrolpi')