wrolpi/wrolpi/files/models.py
Roland ba63170e55
Some checks failed
CodeQL / Analyze (push) Has been cancelled
Controller Tests / controller-tests (push) Has been cancelled
Jest Tests / jest-tests (push) Has been cancelled
Consolidating deletion endpoints.
2026-05-03 19:14:30 -06:00

812 lines
34 KiB
Python

import json
import json
import pathlib
import shutil
import urllib.parse
from datetime import datetime
from typing import List, Type, Optional, Iterable
from sqlalchemy import Column, String, Computed, BigInteger, Boolean, event, Index
from sqlalchemy import types
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import deferred, relationship, Session
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
get_relative_to_media_directory, unique_by_predicate, replace_file
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
from wrolpi.db import get_db_session
from wrolpi.downloader import Download
from wrolpi.errors import FileGroupIsTagged, UnknownFile
from wrolpi.files import indexers
from wrolpi.media_path import MediaPathType
from wrolpi.tags import Tag, TagFile, save_tags_config, sync_tags_directory
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
def into_db(obj):
if isinstance(obj, datetime):
return strftime(obj)
if isinstance(obj, pathlib.Path):
return str(obj)
return obj
def out_of_db(media_directory, obj):
if obj and isinstance(obj, str) and obj.startswith(media_directory):
return pathlib.Path(obj)
if obj and isinstance(obj, str) and obj[0].isdigit():
try:
return strptime_ms(obj)
except Exception:
# Wasn't a datetime after all.
pass
return obj
class FancyJSON(types.TypeDecorator):
"""Converts pathlib.Path to strings when moving into DB, and vice versa.
Converts datetime to ISO strings when moving into DB, and vice versa.
"""
impl = JSONB
def process_bind_param(self, value, dialect):
if value:
# datetime(2020, 1, 1, 0, 0, 0) <-> '2020-01-01T00:00:00'
value = recursive_map(value, into_db)
return value
def process_result_value(self, value, dialect) -> None:
if value:
media_directory = str(get_media_directory())
value = recursive_map(value, lambda i: out_of_db(media_directory, i))
return value
class FileGroup(ModelHelper, Base):
__tablename__ = 'file_group'
__table_args__ = (
Index('file_group_author_idx', 'author'),
Index('file_group_directory_idx', 'directory'),
Index('file_group_download_datetime_idx', 'download_datetime'),
Index('file_group_effective_datetime_idx', 'effective_datetime'),
Index('file_group_length_idx', 'length'),
Index('file_group_mimetype_idx', 'mimetype'),
Index('file_group_model_idx', 'model'),
Index('file_group_modification_datetime_idx', 'modification_datetime'),
Index('file_group_published_datetime_idx', 'published_datetime'),
Index('file_group_published_modified_datetime_idx', 'published_modified_datetime'),
Index('file_group_size_ix', 'size'),
Index('file_group_textsearch_idx', 'textsearch'),
Index('file_group_url_idx', 'url'),
Index('file_group_viewed_idx', 'viewed'),
)
id: int = Column(BigInteger, primary_key=True)
# Directory containing all files in this group (absolute path).
# All paths in `files` and `data` are relative to this directory.
directory: pathlib.Path = Column(MediaPathType, nullable=False)
author = Column(String) # name of the author, maybe even a URL
censored = Column(Boolean) # the file is no longer available for download
data = Column(FancyJSON) # populated by the modeler
download_datetime = Column(TZDateTime) # the date WROLPi downloaded this file.
files = Column(FancyJSON) # populated during discovery
idempotency = Column(TZDateTime) # used to track which files need to be deleted during refresh
indexed = Column(Boolean, default=lambda: False) # wrolpi.files.lib.apply_indexers
length = Column(BigInteger) # video duration, article words, etc.
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
model = Column(String) # "video", "archive", "doc", etc.
modification_datetime = Column(TZDateTime) # the modification date of the file on disk
primary_path: pathlib.Path = Column(MediaPathType, nullable=False, unique=True)
published_datetime = Column(TZDateTime) # the date the creator published this file
published_modified_datetime = Column(TZDateTime) # the date the publisher modified this file
size = Column(BigInteger, default=lambda: 0)
title = Column(String) # user-displayable title
url = Column(String) # the location where this file can be downloaded.
viewed = Column(TZDateTime) # the most recent time a User viewed this file.
# Columns updated by triggers.
# `file_group_effective_datetime_trigger` and `update_effective_datetime` event handler
effective_datetime = Column(TZDateTime) # Equivalent to COALESCE(published_datetime, download_datetime)
tag_files: Iterable[TagFile] = relationship('TagFile', cascade='all')
a_text = deferred(Column(String))
b_text = deferred(Column(String))
c_text = deferred(Column(String))
d_text = deferred(Column(String))
textsearch = deferred(
Column(tsvector, Computed('''
setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char")
''')))
def __repr__(self):
m = f'model={self.model}' if self.model else f'mimetype={self.mimetype}'
return f'<FileGroup id={self.id} {m} primary_path={repr(str(self.primary_path))}>'
def __json__(self) -> dict:
from wrolpi.files.lib import split_path_stem_and_suffix
_, suffix = split_path_stem_and_suffix(self.primary_path)
# Get tag names with error handling
tag_names = []
for tag_file in self.tag_files:
try:
if tag_file.tag is not None:
tag_names.append(tag_file.tag.name)
except AttributeError:
if PYTEST:
raise
# Log the error but continue processing other tags
logger.error(
f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
tags = sorted(tag_names)
d = dict(
author=self.author,
censored=self.censored,
data=self.data,
directory=self.directory,
download_datetime=self.download_datetime,
files=self.my_files(),
id=self.id,
key=self.primary_path,
length=self.length,
mimetype=self.mimetype,
model=self.model,
modified=self.modification_datetime or None,
name=self.primary_path.name,
primary_path=self.primary_path,
published_datetime=self.published_datetime,
published_modified_datetime=self.published_modified_datetime,
size=self.size,
suffix=suffix,
tags=tags,
title=self.title,
url=self.url,
viewed=self.viewed,
)
return d
@property
def name(self) -> str:
if self.title:
return self.title
return self.primary_path.name
@property
def full_primary_path(self) -> pathlib.Path:
"""Returns the complete absolute path to the primary file.
Handles both old format (primary_path is absolute) and new format (directory + relative primary_path).
"""
if self.directory:
# New format: directory + relative filename
return self.directory / self.primary_path
# Old format: primary_path is already absolute
return self.primary_path
def resolve_path(self, relative_path: str | pathlib.Path) -> pathlib.Path:
"""Convert a relative filename to an absolute path using this FileGroup's directory.
Handles both old format (path is already absolute) and new format (relative to directory).
"""
if relative_path is None:
return None
path = pathlib.Path(relative_path) if isinstance(relative_path, str) else relative_path
if path.is_absolute():
# Old format: already absolute
return path
if self.directory:
# New format: resolve relative to directory
return self.directory / path
# Fallback: use primary_path's parent as directory
return self.primary_path.parent / path
def set_viewed(self, viewed: datetime = None) -> datetime:
"""
:param viewed: Used only for testing!
:return: The datetime that was set.
"""
self.viewed = viewed or now()
return self.viewed
def set_tags(self, session: Session, tag_names_or_ids: Iterable[str | int]):
"""Insert or Delete TagFiles as necessary to match the provided tags for this FileGroup."""
tag_names_or_ids = list(tag_names_or_ids)
if tag_names_or_ids and isinstance(tag_names_or_ids[0], str):
tag_ids = {Tag.get_id_by_name(session, i) for i in tag_names_or_ids}
else:
tag_ids = set(tag_names_or_ids)
existing_tag_ids = {i.tag_id for i in self.tag_files}
if new_tag_ids := (tag_ids - existing_tag_ids):
for tag_id in new_tag_ids:
tag_file = TagFile(file_group_id=self.id, tag_id=tag_id)
self.tag_files.append(tag_file)
session.flush(self.tag_files)
if deleted_tags := (existing_tag_ids - tag_ids):
session.query(TagFile).filter(
TagFile.file_group_id == self.id,
TagFile.tag_id.in_(deleted_tags),
).delete(synchronize_session=False)
# Refresh the relationship after deleting
session.expire(self, ['tag_files'])
# Save changes to config.
save_tags_config.activate_switch()
sync_tags_directory.activate_switch()
return self.tag_files
def add_tag(self, session: Session, tag_id: int | str) -> TagFile:
"""Add a tag to this FileGroup. `tag_id` can be an int (tag ID) or str (tag name)."""
if isinstance(tag_id, str):
tag_id = Tag.get_id_by_name(session, tag_id)
tags = [i.tag_id for i in self.tag_files]
tags.append(tag_id)
tag_files = self.set_tags(session, tags)
return next(i for i in tag_files if i.tag_id == tag_id)
def untag(self, session: Session, tag_id: int | str):
"""Remove a tag from this FileGroup. `tag_id` can be an int (tag ID) or str (tag name)."""
if isinstance(tag_id, str):
tag_id = Tag.get_id_by_name(session, tag_id)
tags = [i.tag_id for i in self.tag_files if i.tag_id != tag_id]
self.set_tags(session, tags)
def append_files(self, *paths: pathlib.Path):
"""Add all `paths` to this FileGroup.files.
Paths are stored as filenames only (relative to directory).
Also updates modification_datetime to include the new files.
"""
from wrolpi.dates import from_timestamp
from wrolpi.files.lib import get_mimetype
new_files = list(self.files) if self.files else list()
for path in paths:
# Store just the filename, not the full path
size = path.stat().st_size if path.exists() else None
new_files.append(dict(path=path.name, mimetype=get_mimetype(path), size=size))
self.files = unique_by_predicate(new_files, lambda i: i['path'])
# Update modification_datetime to include the new files' mtimes
if paths:
all_mtimes = [p.stat().st_mtime for p in paths if p.exists()]
if self.modification_datetime:
all_mtimes.append(self.modification_datetime.timestamp())
if all_mtimes:
self.modification_datetime = from_timestamp(max(all_mtimes))
def my_files(self, *mimetypes: str) -> List[dict]:
"""Return all files related to this group that match any of the provided mimetypes.
Paths are resolved to absolute paths using directory.
Returns a new list (does not modify self.files in place).
>>> FileGroup().my_files()
>>> FileGroup().my_files('application/pdf')
>>> FileGroup().my_files('video/')
"""
if not self.files:
logger.error(f'{self} has no files!')
raise ValueError(f'{self} has no files!')
# Create a new list with resolved paths (don't modify self.files in place)
result = []
for file_info in self.files:
# Make a copy of each dict to avoid modifying the original
file_copy = dict(file_info)
path = pathlib.Path(file_info['path'])
if not path.is_absolute() and self.directory:
# New format: relative path, resolve using directory
file_copy['path'] = self.directory / path
else:
# Old format or already absolute: keep as-is
file_copy['path'] = path
result.append(file_copy)
if mimetypes:
result = list(filter(lambda j: any(j['mimetype'].startswith(m) for m in mimetypes), result))
if PYTEST:
# Sort files to avoid random order during testing.
return sorted(result, key=lambda j: j['path'])
return result
def my_paths(self, *mimetypes: str) -> List[pathlib.Path]:
return [i['path'] for i in self.my_files(*mimetypes)]
def my_video_files(self):
"""Return all my Files that are videos."""
return self.my_files('video/')
def my_audio_files(self):
"""Return all my Files that are audio."""
return self.my_files('audio/')
def my_json_files(self):
"""Return all my Files that are JSON."""
return self.my_files('application/json')
def my_poster_files(self) -> List[dict]:
"""Return all my Files that are images."""
return self.my_files('image/')
def my_subtitle_files(self) -> List[dict]:
"""Return all my Files that have text/vtt or text/srt mimetype."""
return self.my_files('text/vtt', 'text/srt')
def my_text_files(self) -> List[dict]:
"""Return all my Files that have a text mimetype. But, do not include subtitle files."""
text_files = self.my_files('text/')
subtitle_paths = [i['path'] for i in self.my_subtitle_files()]
return [i for i in text_files if i['path'] not in subtitle_paths]
def my_epub_files(self) -> List[dict]:
return self.my_files('application/epub')
def my_mobi_files(self) -> List[dict]:
return self.my_files('application/x-mobipocket-ebook')
def my_html_files(self) -> List[dict]:
html_files = self.my_files('text/html', 'application/octet-stream')
html_files = [i for i in html_files if i['path'].name.endswith('.html')]
return html_files
def my_html_paths(self) -> List[pathlib.Path]:
return [i['path'] for i in self.my_html_files()]
def _find_unique_json(self, suffix: str) -> pathlib.Path | None:
"""Find a JSON file ending with `suffix` in this FileGroup's files.
@raise ValueError: If multiple files matching the suffix are found."""
from wrolpi.files.lib import split_path_stem_and_suffix
matches = [f for f in self.my_json_files() if f['path'].name.endswith(suffix)]
if len(matches) > 1:
# Prefer the file whose stem has no extra extension (e.g. video.info.json over video.mp4.info.json).
clean = [f for f in matches if '.' not in split_path_stem_and_suffix(f['path'].name)[0]]
if len(clean) == 1:
duplicates = [f['path'] for f in matches if f not in clean]
logger.warning(f'{self} has duplicate {suffix} files which will be ignored: {duplicates}')
return clean[0]['path']
raise ValueError(f'{self} has multiple {suffix} files: {[f["path"] for f in matches]}')
if matches:
return matches[0]['path']
return None
@property
def info_json_path(self) -> pathlib.Path | None:
"""Find the .info.json file in this FileGroup's files.
@raise ValueError: If multiple .info.json files are found."""
return self._find_unique_json('.info.json')
@property
def readability_json_path(self) -> pathlib.Path | None:
"""Find the .readability.json file in this FileGroup's files.
@raise ValueError: If multiple .readability.json files are found."""
return self._find_unique_json('.readability.json')
@property
def metadata_json_path(self) -> pathlib.Path | None:
"""Find the primary metadata JSON file for this FileGroup.
Returns .info.json if it exists, otherwise .readability.json.
@raise ValueError: If multiple files of either type are found."""
return self.info_json_path or self.readability_json_path
def replace_info_json(self, info_json: dict, format_: bool = True):
"""Write `info_json` to this FileGroup's .info.json file.
Finds the .info.json via self. Creates the file if it doesn't exist.
Raises ValueError if multiple .info.json files are tracked.
"""
json_path = self.info_json_path or self.primary_path.with_suffix('.info.json')
self._write_json(json_path, info_json, format_=format_)
def replace_readability_json(self, data: dict, format_: bool = True):
"""Write `data` to this FileGroup's .readability.json file.
Finds the .readability.json via self. Creates the file if it doesn't exist.
Raises ValueError if multiple .readability.json files are tracked.
"""
json_path = self.readability_json_path or self.primary_path.with_suffix('.readability.json')
self._write_json(json_path, data, format_=format_)
def _write_json(self, json_path: pathlib.Path, data: dict, format_: bool = True):
"""Write `data` as JSON to `json_path` and register the file with this FileGroup."""
if format_:
contents = json.dumps(data, indent=2, sort_keys=True)
else:
contents = json.dumps(data)
replace_file(json_path, contents, missing_ok=True)
# Register the file if not already tracked.
tracked_paths = {f['path'] for f in self.files} if self.files else set()
if json_path.name not in tracked_paths:
self.append_files(json_path)
def update_wrolpi_json(self, data: dict):
"""Read this FileGroup's metadata JSON, merge `data` into its `wrolpi` section, and write back.
Uses .info.json if it exists, otherwise .readability.json.
No-op if no metadata JSON file exists.
"""
json_path = self.metadata_json_path
if not json_path or not json_path.is_file():
return
contents = json.loads(json_path.read_text())
wrolpi_section = contents.get('wrolpi', {})
wrolpi_section.update(data)
contents['wrolpi'] = wrolpi_section
# Write back using the appropriate method based on file type.
if json_path.name.endswith('.readability.json'):
self.replace_readability_json(contents)
else:
self.replace_info_json(contents)
def delete(self, add_to_skip_list: bool = True, force: bool = False):
"""Delete this FileGroup record, and all of its files.
@raise FileGroupIsTagged: If any tags are related to me and force is False."""
if self.tag_files and not force:
raise FileGroupIsTagged(f'Cannot delete {self} because it has tags')
for path in self.my_paths():
path.unlink(missing_ok=True)
if session := Session.object_session(self):
from wrolpi.downloader import download_manager
if self.url and (download := Download.get_by_url(session, self.url)):
download.delete(add_to_skip_list)
elif self.url and add_to_skip_list is True:
download_manager.add_to_skip_list(self.url)
session.delete(self)
@property
def indexer(self) -> Type[indexers.Indexer]:
if not self.mimetype:
raise ValueError(f'Cannot find indexer because {self} does not have a mimetype!')
return indexers.find_indexer(self.mimetype)
def do_index(self):
"""Gather any missing information about this file group. Index the contents of this file using an Indexer."""
try:
# Get the indexer on a separate line for debugging.
indexer = self.indexer
# Only read the contents of the file once.
start = now()
self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self.primary_path)
self.title = self.title or self.primary_path.name
if (total_seconds := (now() - start).total_seconds()) > 1:
logger.info(f'Indexing {self.primary_path} took {total_seconds} seconds')
except Exception as e:
logger.error(f'Failed to index {self}', exc_info=e)
if PYTEST:
raise
@classmethod
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
"""Create a new FileGroup which contains the provided file paths."""
from wrolpi.files.lib import get_primary_file, get_mimetype, sanitize_filename_surrogates
# Sanitize any paths with invalid UTF-8 characters (renames files on disk if needed)
paths = tuple(sanitize_filename_surrogates(p) for p in paths)
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}')
if len(existing_groups) == 0:
# These paths have not been used previously, create a new FileGroup.
file_group = FileGroup()
primary_path = get_primary_file(paths)
file_group.primary_path = primary_path
file_group.append_files(*paths)
session.add(file_group)
elif len(existing_groups) == 1:
# Found one FileGroup with these paths, no need to create a new FileGroup.
file_group = existing_groups[0]
file_group.append_files(*paths)
primary_path = get_primary_file(file_group.my_paths())
else:
# Multiple FileGroups contain these paths as primary.
primary_path = get_primary_file(paths)
file_group: FileGroup = next(filter(lambda i: i.primary_path == primary_path, existing_groups), None)
if not file_group:
file_group = FileGroup.from_paths(session, primary_path)
file_group.merge(existing_groups)
if not isinstance(primary_path, pathlib.Path):
raise ValueError('Cannot create FileGroup without a primary path.')
file_group.directory = primary_path.parent
file_group.primary_path = primary_path
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
file_group.size = sum(i.stat().st_size for i in paths)
file_group.mimetype = get_mimetype(file_group.primary_path)
logger.trace(f'FileGroup.from_paths: {file_group}')
return file_group
@staticmethod
def find_by_id(session: Session, id_: int) -> 'FileGroup':
fg = session.query(FileGroup).filter(FileGroup.id == id_).one_or_none()
if not fg:
raise UnknownFile(f'Unable to find FileGroup with id {id_}')
return fg
@classmethod
def get_by_ids(cls, session: Session, file_group_ids: List[int]) -> List['FileGroup']:
"""Return FileGroups matching the given IDs (no error if some are missing)."""
if not file_group_ids:
return []
return session.query(cls).filter(cls.id.in_(file_group_ids)).all()
@staticmethod
def get_by_path(session: Session, path) -> Optional['FileGroup']:
file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none()
return file_group
@classmethod
def get_by_any_file_path(cls, session: Session, path: pathlib.Path) -> Optional['FileGroup']:
"""Find a FileGroup that contains the given file path (primary or not).
This method first checks if the path is a primary_path, and if not,
searches for a FileGroup where the file is in the files list.
"""
path = pathlib.Path(path)
directory = str(path.parent)
filename = path.name
# First, try to find by primary_path (most common case)
if file_group := cls.get_by_path(session, path):
return file_group
# Search for FileGroups in the same directory that contain this file
file_groups = session.query(FileGroup).filter(FileGroup.directory == directory).all()
for fg in file_groups:
for file_info in fg.files:
if file_info.get('path') == filename:
return fg
return None
@staticmethod
def find_by_path(session: Session, path) -> 'FileGroup':
file_group = FileGroup.get_by_path(session, path)
if not file_group:
raise UnknownFile(f'Unable to find FileGroup with path {path}')
return file_group
@classmethod
def find_by_directory(cls, session: Session, directory: pathlib.Path, recursive: bool = False) -> List['FileGroup']:
"""Find all FileGroups in a directory (optionally recursive).
Uses the indexed `directory` column for efficient lookups.
"""
from sqlalchemy import or_
directory_str = str(directory)
if recursive:
return session.query(cls).filter(
or_(cls.directory == directory_str, cls.directory.like(f'{directory_str}/%'))
).all()
return session.query(cls).filter(cls.directory == directory_str).all()
@property
def tag_names(self) -> List[str]:
result = []
for tag_file in self.tag_files:
try:
if tag_file.tag is not None:
result.append(tag_file.tag.name)
except AttributeError:
if PYTEST:
raise
# Log the error but continue processing other tags
logger.error(
f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
return result
def merge(self, file_groups: List['FileGroup']):
"""Consume the files and Tags of the provided FileGroups and attach them to this FileGroup. Delete the provided
FileGroups."""
session = Session.object_session(self)
collected_files = self.files.copy()
# Track filenames (strings) for proper deduplication
existing_filenames = {f['path'] for f in self.files}
for file_group in file_groups:
if file_group.primary_path == self.primary_path:
# Don't merge myself.
continue
for file in file_group.files:
if file['path'] not in existing_filenames:
collected_files.append(file)
existing_filenames.add(file['path']) # Track newly added
# Move any applied Tags.
for tag_file in file_group.tag_files:
if tag_file.tag is not None and tag_file.tag.name not in self.tag_names:
# Preserve the created at.
self.add_tag(session, tag_file.tag.id).created_at = tag_file.created_at
session.delete(file_group)
self.files = collected_files
def move(self, new_primary_path: pathlib.Path):
"""Move all files in this group to a new location.
With relative paths stored in `files` and `data`, we only need to:
1. Move physical files to new location
2. Update `directory` to point to new parent
3. Update `primary_path` to the new path
The relative filenames in `files` and `data` remain unchanged.
"""
from wrolpi.files.lib import split_path_stem_and_suffix, glob_shared_stem, split_file_name_words
if new_primary_path.exists():
raise FileExistsError(f'Cannot move {self} to {new_primary_path} because it already exists.')
# Adopt any new files when a FileGroup has multiple files.
if len(self.my_files()) > 1:
self.append_files(*glob_shared_stem(self.primary_path))
new_name, _ = split_path_stem_and_suffix(new_primary_path, full=True)
new_directory = new_primary_path.parent
# Get current files with resolved absolute paths for moving
current_files = self.my_files()
# Ensure that all destination files do not yet exist before move.
for file in current_files:
_, suffix = split_path_stem_and_suffix(file['path'])
new_path = pathlib.Path(f'{new_name}{suffix}')
if new_path.exists():
raise FileExistsError(f'Cannot move {self} to {new_path} because it already exists.')
# Move physical files and collect new relative file entries
new_files = []
for file in current_files:
old_path = file['path'] # Already resolved to absolute by my_files()
_, suffix = split_path_stem_and_suffix(old_path)
new_path = pathlib.Path(f'{new_name}{suffix}')
if old_path.is_file():
shutil.move(old_path, new_path)
# Store just the filename (relative path)
new_files.append({'path': new_path.name, 'mimetype': file['mimetype']})
logger.debug(f'Moved FileGroup: {self.primary_path} -> {new_primary_path}')
# Check if filename changed before updating primary_path
old_name = self.primary_path.name
filename_changed = old_name != new_primary_path.name
# Update the FileGroup with new directory and relative files
self.files = new_files
self.directory = new_directory
self.primary_path = new_primary_path
# Update title and a_text only if filename changed
if filename_changed:
if self.title == old_name:
self.title = new_primary_path.name
self.a_text = split_file_name_words(new_primary_path.name)
# Note: `data` now contains relative paths that don't need updating!
# The filenames stay the same, only the directory changes.
# Only re-index if there's no data yet (new file that hasn't been modeled).
if not self.data:
self.indexed = False
# Note: Caller is responsible for flushing/committing
@property
def location(self):
"""Returns the URL that the FileGroup can be previewed."""
klass = self.get_model_class()
if klass:
# Return the model location when possible.
with get_db_session() as session:
instance = klass.get_by_path(session, self.primary_path)
if instance and hasattr(instance, 'location'):
return instance.location
parent = str(get_relative_to_media_directory(self.directory))
preview = str(get_relative_to_media_directory(self.primary_path))
if parent == '.':
# File is in the top of the media directory, App already shows top directory open.
query = urllib.parse.urlencode(dict(preview=str(preview)))
else:
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
return f'/files?{query}'
def get_model_class(self) -> Optional[Type[ModelHelper]]:
"""Return the class that is suitable for this FileGroup (usually based on mimetype), if any."""
from modules.videos.models import Video
from modules.archive.models import Archive
from modules.docs.models import Doc
from modules.zim.models import Zim
if Video.can_model(self):
return Video
elif Doc.can_model(self):
return Doc
elif Archive.can_model(self):
return Archive
elif Zim.can_model(self):
return Zim
def get_model_record(self, session: Session) -> Optional[ModelHelper]:
"""Return this FileGroup's model record (Video/Archive/Doc/Zim) using the given session, or None."""
klass = self.get_model_class()
return klass.get_by_path(session, self.primary_path) if klass else None
def do_model(self, session: Session) -> Optional[ModelHelper]:
"""Get/Create the Model record (Video/Archive/etc.) for this FileGroup, if any."""
if model_class := self.get_model_class():
# Always call do_model() to ensure FileGroup.data is updated when files change.
# The model's do_model() should be idempotent (safe to call multiple times).
model = model_class.do_model(session, self)
logger.debug(f'modeled: {self} {model}')
return model
# Index the file if no models are available.
logger.debug(f'no model class: {self}')
self.do_index()
self.indexed = True
def get_tag_directory_paths_map(self) -> dict[pathlib.Path, str]:
"""Return all links that should exist for this FileGroup in the Tags Directory."""
tag_names = self.tag_names
if not tag_names:
raise RuntimeError(f'Did not get any tag names for {self}')
try:
tag_names_str = ', '.join(sorted(tag_names))
files = dict()
for file in self.my_paths():
files[file] = f'{tag_names_str}/{file.name}'
return files
except Exception as e:
logger.error(f"Error in get_tag_directory_paths_map for {self}: {e}")
raise
@event.listens_for(FileGroup, 'before_insert')
@event.listens_for(FileGroup, 'before_update')
def update_effective_datetime(mapper, connection, target):
target.effective_datetime = target.published_datetime or target.download_datetime
class Directory(ModelHelper, Base):
"""A representation of a file directory in the media directory."""
__tablename__ = 'directory'
__table_args__ = (
Index('directory_name_idx', 'name'),
)
path: pathlib.Path = Column(MediaPathType, primary_key=True)
name: str = Column(String, nullable=False)
idempotency = Column(TZDateTime, default=lambda: now())
def __json__(self) -> dict:
d = dict(
path=self.path,
name=self.name,
)
return d
def __repr__(self):
return f'<Directory path={repr(str(self.path))}>'