Linking to uploaded file on download after a user uploads a file with the same URL.

modules/archive/__init__.py

@@ -107,6 +107,25 @@ archive_downloader = ArchiveDownloader()
 
 
 def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
+    """
+    Models an Archive from a given FileGroup.
+
+    This function takes in a FileGroup and attempts to create an Archive object.
+    It does this by checking for the presence of HTML files within the FileGroup,
+    determining if any of these are SingleFiles, and then extracting relevant data
+    (title, contents) from either the JSON or text readability files.
+
+    Args:
+        file_group: The FileGroup to model as an Archive.
+        session: An optional database session for committing changes.
+
+    Returns:
+        An Archive object representing the modeled archive.
+
+    Raises:
+        InvalidArchive: If no HTML SingleFile is found in the FileGroup, or if
+            any other error occurs during modeling.
+    """
     file_group_id = file_group.id
     if not file_group_id:
         session.flush([file_group])

modules/videos/test/test_lib.py

@@ -315,3 +315,14 @@ def test_browser_profile_to_yt_dlp_arg(file_name, expected):
     """
     file_name = pathlib.Path(file_name)
     assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected
+
+
+def test_video_location(test_session, video_factory, channel_factory):
+    # Video location without a channel, it opens the preview.
+    video1 = video_factory(title='vid1')
+    assert video1.location == '/videos/video/1'
+
+    # Video location with a channel.
+    channel = channel_factory(name='ChannelName')
+    video2 = video_factory(title='vid2', channel_id=channel.id)
+    assert video2.location == '/videos/channel/1/video/2'

wrolpi/common.py

@@ -24,7 +24,7 @@ from functools import wraps
 from itertools import islice, filterfalse, tee
 from multiprocessing.managers import DictProxy
 from pathlib import Path
-from types import GeneratorType
+from types import GeneratorType, MappingProxyType
 from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set
 from urllib.parse import urlparse, urlunsplit
 
@@ -445,7 +445,7 @@ class ConfigFile:
         self.width = self.width or 90
 
         # Do not load a global config on import while testing.  A global instance should never be used for testing.
-        self._config = self.default_config.copy()
+        self._config = deepcopy({k: v for k, v in self.default_config.items()})
 
         if self._config['version'] != 0:
             raise RuntimeError('Configs should start at version 0')
@@ -1921,19 +1921,17 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
     source = json.dumps([{'content': i} for i in entries])
     with get_db_curs() as curs:
         stmt = '''
-        WITH vectored AS (
-            -- Convert the "source" json to a recordset.
-            with source as (select * from json_to_recordset(%s::json) AS (content TEXT))
-            select
-                to_tsvector('english'::regconfig, source.content) AS vector,
-                source.content
-            from source
-        )
-        SELECT
-            ts_headline(vectored.content, websearch_to_tsquery(%s), 'MaxFragments=10, MaxWords=8, MinWords=7'),
-            ts_rank(vectored.vector, websearch_to_tsquery(%s))
-        FROM vectored
-        '''
+               WITH vectored AS (
+                   -- Convert the "source" json to a recordset.
+                   with source as (select * from json_to_recordset(%s::json) AS (content TEXT))
+                   select to_tsvector('english'::regconfig, source.content) AS vector,
+                          source.content
+                   from source)
+               SELECT ts_headline(vectored.content, websearch_to_tsquery(%s),
+                                  'MaxFragments=10, MaxWords=8, MinWords=7'),
+                      ts_rank(vectored.vector, websearch_to_tsquery(%s))
+               FROM vectored \
+               '''
         curs.execute(stmt, [source, search_str, search_str])
         headlines = [tuple(i) for i in curs.fetchall()]
 
@@ -1951,12 +1949,11 @@ async def search_other_estimates(tag_names: List[str]) -> dict:
 
     with get_db_curs() as curs:
         stmt = '''
-            SELECT
-                COUNT(c.id)
-            FROM channel c
-            LEFT OUTER JOIN public.tag t on t.id = c.tag_id
-            WHERE t.name = %(tag_name)s
-        '''
+               SELECT COUNT(c.id)
+               FROM channel c
+                        LEFT OUTER JOIN public.tag t on t.id = c.tag_id
+               WHERE t.name = %(tag_name)s \
+               '''
         # TODO handle multiple tags
         params = dict(tag_name=tag_names[0])
         curs.execute(stmt, params)

wrolpi/downloader.py

@@ -1116,11 +1116,13 @@ class DownloadManager:
 
     @staticmethod
     def add_to_skip_list(*urls: str):
-        get_download_manager_config().skip_urls = list(set(get_download_manager_config().skip_urls) | set(urls))
+        get_download_manager_config().skip_urls = sorted(list(set(get_download_manager_config().skip_urls) | set(urls)))
 
     @staticmethod
     def remove_from_skip_list(url: str):
-        get_download_manager_config().skip_urls = [i for i in get_download_manager_config().skip_urls if i != url]
+        skip_urls = get_download_manager_config().skip_urls
+        if url in skip_urls:
+            get_download_manager_config().skip_urls = [i for i in get_download_manager_config().skip_urls if i != url]
 
 
 # The global DownloadManager.  This should be used everywhere!

wrolpi/files/lib.py

@@ -1001,19 +1001,19 @@ def get_file_statistics():
         curs.execute('''
                      SELECT
                          -- All items in file_group.files are real individual files.
-                         SUM(json_array_length(files)) AS                                                                             "total_count",
-                         COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS                                          "pdf_count",
-                         COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS                                          "zip_count",
-                         COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS                                               "video_count",
-                         COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS                                               "image_count",
-                         COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS                                               "audio_count",
+                         SUM(json_array_length(files))                                                                             AS "total_count",
+                         COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf')                                          AS "pdf_count",
+                         COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip')                                          AS "zip_count",
+                         COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%')                                               AS "video_count",
+                         COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%')                                               AS "image_count",
+                         COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%')                                               AS "audio_count",
                          COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype =
                                                                                                  'application/x-mobipocket-ebook') AS "ebook_count",
-                         (SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS                                             "tagged_files",
-                         (SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS                                                   "tagged_zims",
-                         (SELECT COUNT(*) FROM tag) AS                                                                                "tags_count",
-                         SUM(size)::BIGINT AS                                                                                         "total_size",
-                         (SELECT COUNT(*) FROM archive) AS                                                                            archive_count
+                         (SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file)                                             AS "tagged_files",
+                         (SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim)                                                   AS "tagged_zims",
+                         (SELECT COUNT(*) FROM tag)                                                                                AS "tags_count",
+                         SUM(size)::BIGINT                                                                                         AS "total_size",
+                         (SELECT COUNT(*) FROM archive)                                                                            AS archive_count
                      FROM file_group
                      ''')
         statistics = dict(curs.fetchall()[0])
@@ -1610,7 +1610,8 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
         if PYTEST:
             raise
 
-    # If user uploads a file, then remove it from the download skip list so comments can be downloaded.
+    # If user uploads a file, then remove it from the download skip list so comments can be downloaded.  Modify the
+    # download (if any) so that the user can click on it and view the uploaded file.
     if url := file_group.url if file_group and file_group.url else None:
         if download_manager.is_skipped(url):
             download_manager.remove_from_skip_list(url)
@@ -1618,7 +1619,9 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
             if download := Download.get_by_url(url, session):
                 # Mark download as completed
                 download.complete()
-                download.location = file_group.location
+                # Link the download to the location of the uploaded file.
+                model = file_group.get_model_record()
+                download.location = model.location if model else file_group.location
                 session.commit()
 
     upsert_directories([], file.parents)

wrolpi/files/models.py

@@ -13,7 +13,7 @@ from sqlalchemy.orm import deferred, relationship, Session
 from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
     get_relative_to_media_directory, unique_by_predicate
 from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
-from wrolpi.db import optional_session
+from wrolpi.db import optional_session, get_db_session
 from wrolpi.downloader import Download
 from wrolpi.errors import FileGroupIsTagged, UnknownFile
 from wrolpi.files import indexers
@@ -119,7 +119,8 @@ class FileGroup(ModelHelper, Base):
                 if PYTEST:
                     raise
                 # Log the error but continue processing other tags
-                logger.error(f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
+                logger.error(
+                    f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
         tags = sorted(tag_names)
         d = dict(
             author=self.author,
@@ -393,7 +394,8 @@ class FileGroup(ModelHelper, Base):
                 if PYTEST:
                     raise
                 # Log the error but continue processing other tags
-                logger.error(f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
+                logger.error(
+                    f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
         return result
 
     def merge(self, file_groups: List['FileGroup']):
@@ -464,6 +466,14 @@ class FileGroup(ModelHelper, Base):
     @property
     def location(self):
         """Returns the URL that the FileGroup can be previewed."""
+        klass = self.get_model_class()
+        if klass:
+            # Return the model location when possible.
+            with get_db_session() as session:
+                instance = klass.get_by_path(self.primary_path, session)
+                if instance and hasattr(instance, 'location'):
+                    return instance.location
+
         parent = str(get_relative_to_media_directory(self.primary_path.parent))
         preview = str(get_relative_to_media_directory(self.primary_path))
         if parent == '.':
@@ -489,6 +499,13 @@ class FileGroup(ModelHelper, Base):
         elif Zim.can_model(self):
             return Zim
 
+    def get_model_record(self) -> Optional[ModelHelper]:
+        klass = self.get_model_class()
+        if klass:
+            with get_db_session() as session:
+                return klass.get_by_path(self.primary_path, session)
+        return None
+
     def do_model(self, session: Session) -> Optional[ModelHelper]:
         """Get/Create the Model record (Video/Archive/etc.) for this FileGroup, if any."""
         if model_class := self.get_model_class():

wrolpi/files/test/test_api.py

@@ -671,9 +671,12 @@ async def test_post_upload(test_session, async_client, test_directory, make_file
     assert not file_group.tag_names, 'No tags this time.'
     assert file_group.indexed, 'File should be indexed after upload.'
     assert file_group.model, 'File should be modeled'
+    assert isinstance(file_group.get_model_record(), Video)
+    assert file_group.location == '/videos/video/2'
     # Video was modeled, but has no Channel.
     video: Video = test_session.query(Video).one()
     assert video.video_path
+    assert video.file_group == file_group
     assert not video.info_json_path
     assert not video.channel and not video.channel_id