From 76a454ea34b73a019632987f77a761d453501ea1 Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 11 Aug 2025 16:54:10 -0600 Subject: [PATCH] Linking to uploaded file on download after a user uploads a file with the same URL. --- modules/archive/__init__.py | 19 ++++++++++++++++ modules/videos/test/test_lib.py | 11 ++++++++++ wrolpi/common.py | 39 +++++++++++++++------------------ wrolpi/downloader.py | 6 +++-- wrolpi/files/lib.py | 29 +++++++++++++----------- wrolpi/files/models.py | 23 ++++++++++++++++--- wrolpi/files/test/test_api.py | 3 +++ 7 files changed, 91 insertions(+), 39 deletions(-) diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index e7d63fdf..2ced4067 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -107,6 +107,25 @@ archive_downloader = ArchiveDownloader() def model_archive(file_group: FileGroup, session: Session = None) -> Archive: + """ + Models an Archive from a given FileGroup. + + This function takes in a FileGroup and attempts to create an Archive object. + It does this by checking for the presence of HTML files within the FileGroup, + determining if any of these are SingleFiles, and then extracting relevant data + (title, contents) from either the JSON or text readability files. + + Args: + file_group: The FileGroup to model as an Archive. + session: An optional database session for committing changes. + + Returns: + An Archive object representing the modeled archive. + + Raises: + InvalidArchive: If no HTML SingleFile is found in the FileGroup, or if + any other error occurs during modeling. + """ file_group_id = file_group.id if not file_group_id: session.flush([file_group]) diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 182803a5..c01508cb 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -315,3 +315,14 @@ def test_browser_profile_to_yt_dlp_arg(file_name, expected): """ file_name = pathlib.Path(file_name) assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected + + +def test_video_location(test_session, video_factory, channel_factory): + # Video location without a channel, it opens the preview. + video1 = video_factory(title='vid1') + assert video1.location == '/videos/video/1' + + # Video location with a channel. + channel = channel_factory(name='ChannelName') + video2 = video_factory(title='vid2', channel_id=channel.id) + assert video2.location == '/videos/channel/1/video/2' diff --git a/wrolpi/common.py b/wrolpi/common.py index c682c4a4..c598b7a5 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -24,7 +24,7 @@ from functools import wraps from itertools import islice, filterfalse, tee from multiprocessing.managers import DictProxy from pathlib import Path -from types import GeneratorType +from types import GeneratorType, MappingProxyType from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set from urllib.parse import urlparse, urlunsplit @@ -445,7 +445,7 @@ class ConfigFile: self.width = self.width or 90 # Do not load a global config on import while testing. A global instance should never be used for testing. - self._config = self.default_config.copy() + self._config = deepcopy({k: v for k, v in self.default_config.items()}) if self._config['version'] != 0: raise RuntimeError('Configs should start at version 0') @@ -1921,19 +1921,17 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl source = json.dumps([{'content': i} for i in entries]) with get_db_curs() as curs: stmt = ''' - WITH vectored AS ( - -- Convert the "source" json to a recordset. - with source as (select * from json_to_recordset(%s::json) AS (content TEXT)) - select - to_tsvector('english'::regconfig, source.content) AS vector, - source.content - from source - ) - SELECT - ts_headline(vectored.content, websearch_to_tsquery(%s), 'MaxFragments=10, MaxWords=8, MinWords=7'), - ts_rank(vectored.vector, websearch_to_tsquery(%s)) - FROM vectored - ''' + WITH vectored AS ( + -- Convert the "source" json to a recordset. + with source as (select * from json_to_recordset(%s::json) AS (content TEXT)) + select to_tsvector('english'::regconfig, source.content) AS vector, + source.content + from source) + SELECT ts_headline(vectored.content, websearch_to_tsquery(%s), + 'MaxFragments=10, MaxWords=8, MinWords=7'), + ts_rank(vectored.vector, websearch_to_tsquery(%s)) + FROM vectored \ + ''' curs.execute(stmt, [source, search_str, search_str]) headlines = [tuple(i) for i in curs.fetchall()] @@ -1951,12 +1949,11 @@ async def search_other_estimates(tag_names: List[str]) -> dict: with get_db_curs() as curs: stmt = ''' - SELECT - COUNT(c.id) - FROM channel c - LEFT OUTER JOIN public.tag t on t.id = c.tag_id - WHERE t.name = %(tag_name)s - ''' + SELECT COUNT(c.id) + FROM channel c + LEFT OUTER JOIN public.tag t on t.id = c.tag_id + WHERE t.name = %(tag_name)s \ + ''' # TODO handle multiple tags params = dict(tag_name=tag_names[0]) curs.execute(stmt, params) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 70011ac8..e701ef27 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1116,11 +1116,13 @@ class DownloadManager: @staticmethod def add_to_skip_list(*urls: str): - get_download_manager_config().skip_urls = list(set(get_download_manager_config().skip_urls) | set(urls)) + get_download_manager_config().skip_urls = sorted(list(set(get_download_manager_config().skip_urls) | set(urls))) @staticmethod def remove_from_skip_list(url: str): - get_download_manager_config().skip_urls = [i for i in get_download_manager_config().skip_urls if i != url] + skip_urls = get_download_manager_config().skip_urls + if url in skip_urls: + get_download_manager_config().skip_urls = [i for i in get_download_manager_config().skip_urls if i != url] # The global DownloadManager. This should be used everywhere! diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 9ca43aab..2615385c 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1001,19 +1001,19 @@ def get_file_statistics(): curs.execute(''' SELECT -- All items in file_group.files are real individual files. - SUM(json_array_length(files)) AS "total_count", - COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count", - COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count", - COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count", - COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count", - COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count", + SUM(json_array_length(files)) AS "total_count", + COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count", + COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count", + COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count", + COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count", + COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count", COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype = 'application/x-mobipocket-ebook') AS "ebook_count", - (SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files", - (SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims", - (SELECT COUNT(*) FROM tag) AS "tags_count", - SUM(size)::BIGINT AS "total_size", - (SELECT COUNT(*) FROM archive) AS archive_count + (SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files", + (SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims", + (SELECT COUNT(*) FROM tag) AS "tags_count", + SUM(size)::BIGINT AS "total_size", + (SELECT COUNT(*) FROM archive) AS archive_count FROM file_group ''') statistics = dict(curs.fetchall()[0]) @@ -1610,7 +1610,8 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> if PYTEST: raise - # If user uploads a file, then remove it from the download skip list so comments can be downloaded. + # If user uploads a file, then remove it from the download skip list so comments can be downloaded. Modify the + # download (if any) so that the user can click on it and view the uploaded file. if url := file_group.url if file_group and file_group.url else None: if download_manager.is_skipped(url): download_manager.remove_from_skip_list(url) @@ -1618,7 +1619,9 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> if download := Download.get_by_url(url, session): # Mark download as completed download.complete() - download.location = file_group.location + # Link the download to the location of the uploaded file. + model = file_group.get_model_record() + download.location = model.location if model else file_group.location session.commit() upsert_directories([], file.parents) diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 29a60767..79120097 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -13,7 +13,7 @@ from sqlalchemy.orm import deferred, relationship, Session from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \ get_relative_to_media_directory, unique_by_predicate from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime -from wrolpi.db import optional_session +from wrolpi.db import optional_session, get_db_session from wrolpi.downloader import Download from wrolpi.errors import FileGroupIsTagged, UnknownFile from wrolpi.files import indexers @@ -119,7 +119,8 @@ class FileGroup(ModelHelper, Base): if PYTEST: raise # Log the error but continue processing other tags - logger.error(f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}") + logger.error( + f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}") tags = sorted(tag_names) d = dict( author=self.author, @@ -393,7 +394,8 @@ class FileGroup(ModelHelper, Base): if PYTEST: raise # Log the error but continue processing other tags - logger.error(f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}") + logger.error( + f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}") return result def merge(self, file_groups: List['FileGroup']): @@ -464,6 +466,14 @@ class FileGroup(ModelHelper, Base): @property def location(self): """Returns the URL that the FileGroup can be previewed.""" + klass = self.get_model_class() + if klass: + # Return the model location when possible. + with get_db_session() as session: + instance = klass.get_by_path(self.primary_path, session) + if instance and hasattr(instance, 'location'): + return instance.location + parent = str(get_relative_to_media_directory(self.primary_path.parent)) preview = str(get_relative_to_media_directory(self.primary_path)) if parent == '.': @@ -489,6 +499,13 @@ class FileGroup(ModelHelper, Base): elif Zim.can_model(self): return Zim + def get_model_record(self) -> Optional[ModelHelper]: + klass = self.get_model_class() + if klass: + with get_db_session() as session: + return klass.get_by_path(self.primary_path, session) + return None + def do_model(self, session: Session) -> Optional[ModelHelper]: """Get/Create the Model record (Video/Archive/etc.) for this FileGroup, if any.""" if model_class := self.get_model_class(): diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index fc0e2fb0..6fd1078d 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -671,9 +671,12 @@ async def test_post_upload(test_session, async_client, test_directory, make_file assert not file_group.tag_names, 'No tags this time.' assert file_group.indexed, 'File should be indexed after upload.' assert file_group.model, 'File should be modeled' + assert isinstance(file_group.get_model_record(), Video) + assert file_group.location == '/videos/video/2' # Video was modeled, but has no Channel. video: Video = test_session.query(Video).one() assert video.video_path + assert video.file_group == file_group assert not video.info_json_path assert not video.channel and not video.channel_id