wrolpi/modules/archive/test/test_api.py

593 lines
29 KiB
Python
Raw Normal View History

import json
from http import HTTPStatus
from unittest.mock import patch
import pytest
from modules.archive import lib, Archive
from wrolpi.common import get_relative_to_media_directory
from wrolpi.test.common import skip_circleci, skip_macos
async def check_results(async_client, data, ids):
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
if not response.status_code == HTTPStatus.OK:
raise AssertionError(str(response.json))
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
if ids:
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
file_groups = response.json['file_groups']
assert file_groups, f'Expected {len(ids)} archives but did not receive any'
assert [i['id'] for i in response.json['file_groups']] == ids, \
f'{response.json["file_groups"][0]["id"]}..{response.json["file_groups"][-1]["id"]}'
@pytest.mark.asyncio
async def test_archives_search_order(test_session, archive_directory, archive_factory, async_client):
"""Search using all orders."""
archive_factory('example.com', 'https://example.com/one', 'my archive', 'foo bar qux')
test_session.commit()
for order_by in lib.ARCHIVE_ORDERS:
data = {'search_str': 'foo', 'order_by': order_by}
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
if not response.status_code == HTTPStatus.OK:
raise AssertionError(str(response.json))
2023-05-16 14:59:35 -06:00
data = {'search_str': None, 'order_by': order_by}
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
2023-05-16 14:59:35 -06:00
if not response.status_code == HTTPStatus.OK:
raise AssertionError(str(response.json))
@pytest.mark.asyncio
async def test_archives_search(test_session, archive_directory, archive_factory, async_client):
"""Archives can be searched by their title and their contents."""
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
# Search with no archives.
await check_results(async_client, {'search_str': 'foo'}, [])
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
archive_factory('example.com', 'https://example.com/one', 'my archive', 'foo bar qux')
archive_factory('example.com', 'https://example.com/one', 'other archive', 'foo baz qux qux')
archive_factory('example.org', title='archive third', contents='baz qux qux qux')
archive_factory('example.org') # has no contents
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
test_session.commit()
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
# Contents are d_text, so a deep search is required to match them.
# 1 and 2 contain "foo".
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
data = {'search_str': 'foo', 'deep': True}
await check_results(async_client, data, [2, 1])
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
# The default (fast) search does not match contents.
data = {'search_str': 'foo'}
await check_results(async_client, data, [])
# 2 and 3 contain "baz".
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
data = {'search_str': 'baz', 'deep': True}
await check_results(async_client, data, [3, 2])
# 1 contains "bar".
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
data = {'search_str': 'bar', 'deep': True}
await check_results(async_client, data, [1, ])
# No archives contain "huzzah"
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
data = {'search_str': 'huzzah', 'deep': True}
await check_results(async_client, data, [])
# Only 3 contains "baz" and is in domain "example.org"
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
data = {'search_str': 'baz', 'domain': 'example.org', 'deep': True}
await check_results(async_client, data, [3, ])
Postgres -> SQLite hard cutover: the database moves into the media directory. The database now lives at <media>/config/wrolpi.db so a library (files, configs, AND database) travels with the drive between WROLPi systems. Core: - wrolpi/db.py: lazy SQLite engine factory (WAL, busy_timeout=30s, foreign_keys=ON); get_db_curs yields a plain sqlite3.Cursor; native sqlite3 paramstyle everywhere (no compatibility shims); global sqlite3 adapters keep raw-SQL datetime formats identical to the ORM's. - wrolpi/fts.py: FTS5 external-content search replacing tsvector/GIN. Injection-proof websearch->FTS5 translator, bm25(10,4,2,1) reproducing A/B/C/D weighting, fast(abc)/deep column filtering, snippet headlines. Stemming contract pinned by dedicated tests (watch/watches/watched/ watching are interchangeable). - wrolpi/schema_ddl.py: SQLite triggers shared verbatim by the alembic baseline and test DBs; fixes the channel video_count DELETE/UPDATE bugs and a latent total_size bug (it used the video id as channel id). - wrolpi/db_bootstrap.py: startup auto-create/migrate; refuses databases written by a newer WROLPi (db_version_mismatch); SQLite 3.40 floor; network-filesystem refusal; never creates/migrates/trusts a database beneath an unmounted production media directory (shadow-DB guard). - Alembic: 85 Postgres-era migrations squashed into one baseline; revision ids are dates (2026_07_10_1350) so filenames self-sort and alembic_version says when a schema was created; render_as_batch=True. - All raw Postgres SQL rewritten (ANY->json_each IN, array_agg @> -> HAVING COUNT(DISTINCT), unnest->executemany/UPDATE..FROM, generate_series histograms->strftime GROUP BY, DISTINCT ON-> row_number(), writable CTEs->two statements, jsonb_*->json_*, explicit NULLS FIRST/LAST, LIMIT-before-OFFSET). Simple statements converted to SQLAlchemy instead (flag upsert, download retry/summary/tag checks, channel-by-tag estimate, Directory delete). - Config imports count skipped entries; background dumps are blocked after a lossy import so partial imports can never overwrite YAML configs (they are the sole source of truth for rebuilds now). - Search tolerates explicit JSON nulls for limit/offset (Postgres treated OFFSET NULL as 0; SQLite errors). - file_group.id/doc_section.id render as INTEGER PRIMARY KEY on SQLite (BIGINT is not a rowid alias; FTS5 content_rowid requires it). Ops teardown: docker-compose db service, CI postgres sidecars, initialize_api_db.sh, controller postgresql service, psycopg2-binary all removed. upgrade.sh stops/disables postgresql (data is NOT deleted; `systemctl enable --now postgresql` restores it); systemd units no longer Want postgresql.service. Existing-user upgrade: first boot creates+migrates the DB, configs import (tags recreate their FileGroups from disk), a refresh rebuilds the library index. QA'd on the 10.0.0.8 RPi: rebuild matched the Postgres baseline exactly; survived an uncontrolled hard reboot with clean integrity checks. Accepted behavior changes: FTS5 has no stopwords; bm25 length normalization shifts some rank orders; fast-path d_headline is un-highlighted; all-negative queries match nothing. Deferred: Debian/trixie package lists (incl. sqlite3 CLI), Cypress re-baseline, VACUUM INTO snapshots, exFAT/NTFS warning, auto-kicked rebuild refresh with progress UX. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-12 13:59:20 -06:00
# 1's title contains "my". Postgres ignored this stopword; FTS5 has no stopwords, so it matches.
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
data = {'search_str': 'my', 'deep': True}
Postgres -> SQLite hard cutover: the database moves into the media directory. The database now lives at <media>/config/wrolpi.db so a library (files, configs, AND database) travels with the drive between WROLPi systems. Core: - wrolpi/db.py: lazy SQLite engine factory (WAL, busy_timeout=30s, foreign_keys=ON); get_db_curs yields a plain sqlite3.Cursor; native sqlite3 paramstyle everywhere (no compatibility shims); global sqlite3 adapters keep raw-SQL datetime formats identical to the ORM's. - wrolpi/fts.py: FTS5 external-content search replacing tsvector/GIN. Injection-proof websearch->FTS5 translator, bm25(10,4,2,1) reproducing A/B/C/D weighting, fast(abc)/deep column filtering, snippet headlines. Stemming contract pinned by dedicated tests (watch/watches/watched/ watching are interchangeable). - wrolpi/schema_ddl.py: SQLite triggers shared verbatim by the alembic baseline and test DBs; fixes the channel video_count DELETE/UPDATE bugs and a latent total_size bug (it used the video id as channel id). - wrolpi/db_bootstrap.py: startup auto-create/migrate; refuses databases written by a newer WROLPi (db_version_mismatch); SQLite 3.40 floor; network-filesystem refusal; never creates/migrates/trusts a database beneath an unmounted production media directory (shadow-DB guard). - Alembic: 85 Postgres-era migrations squashed into one baseline; revision ids are dates (2026_07_10_1350) so filenames self-sort and alembic_version says when a schema was created; render_as_batch=True. - All raw Postgres SQL rewritten (ANY->json_each IN, array_agg @> -> HAVING COUNT(DISTINCT), unnest->executemany/UPDATE..FROM, generate_series histograms->strftime GROUP BY, DISTINCT ON-> row_number(), writable CTEs->two statements, jsonb_*->json_*, explicit NULLS FIRST/LAST, LIMIT-before-OFFSET). Simple statements converted to SQLAlchemy instead (flag upsert, download retry/summary/tag checks, channel-by-tag estimate, Directory delete). - Config imports count skipped entries; background dumps are blocked after a lossy import so partial imports can never overwrite YAML configs (they are the sole source of truth for rebuilds now). - Search tolerates explicit JSON nulls for limit/offset (Postgres treated OFFSET NULL as 0; SQLite errors). - file_group.id/doc_section.id render as INTEGER PRIMARY KEY on SQLite (BIGINT is not a rowid alias; FTS5 content_rowid requires it). Ops teardown: docker-compose db service, CI postgres sidecars, initialize_api_db.sh, controller postgresql service, psycopg2-binary all removed. upgrade.sh stops/disables postgresql (data is NOT deleted; `systemctl enable --now postgresql` restores it); systemd units no longer Want postgresql.service. Existing-user upgrade: first boot creates+migrates the DB, configs import (tags recreate their FileGroups from disk), a refresh rebuilds the library index. QA'd on the 10.0.0.8 RPi: rebuild matched the Postgres baseline exactly; survived an uncontrolled hard reboot with clean integrity checks. Accepted behavior changes: FTS5 has no stopwords; bm25 length normalization shifts some rank orders; fast-path d_headline is un-highlighted; all-negative queries match nothing. Deferred: Debian/trixie package lists (incl. sqlite3 CLI), Cypress re-baseline, VACUUM INTO snapshots, exFAT/NTFS warning, auto-kicked rebuild refresh with progress UX. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-12 13:59:20 -06:00
await check_results(async_client, data, [1, ])
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
# 3's title contains "third", found by the default (fast) search and the deep search.
data = {'search_str': 'third'}
await check_results(async_client, data, [3, ])
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
data = {'search_str': 'third', 'deep': True}
await check_results(async_client, data, [3, ])
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
# All contents contain "qux", but they contain different amounts. They are ordered by the amount.
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
data = {'search_str': 'qux', 'deep': True}
await check_results(async_client, data, [3, 2, 1])
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
data = {'search_str': 'qux', 'order_by': 'bad order_by'}
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_search_archive_tags(test_session, async_client, archive_factory, tag_factory):
"""Tagged Archives can be searched."""
tag = await tag_factory()
archive_factory(domain='example.com', tag_names=[tag.name, ])
test_session.commit()
content = {'search_str': '', 'domain': 'example.com', 'tag_names': [tag.name, ]}
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups']
assert response.json['totals']['file_groups'] == 1
content = {'search_str': '', 'domain': 'example.com', 'tag_names': ['does not exist', ]}
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == []
assert response.json['totals']['file_groups'] == 0
@pytest.mark.asyncio
async def test_archives_search_headline(test_session, archive_directory, archive_factory, async_client):
2023-04-12 19:07:16 -06:00
"""Headlines can be requested."""
archive_factory('example.com', 'https://example.com/one', 'my archive', 'foo bar qux')
archive_factory('example.com', 'https://example.com/one', 'other archive', 'foo baz qux qux')
archive_factory('example.org', title='archive third', contents='baz qux qux qux')
archive_factory('example.org') # has no contents
test_session.commit()
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
# "foo" is only in the contents (d_text), so a deep search is required to match it.
content = dict(search_str='foo', deep=True)
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
2023-04-12 19:07:16 -06:00
assert response.status_code == HTTPStatus.OK
# Headlines are only fetched if requested.
assert response.json['file_groups'][0]['d_headline'] is None
assert response.json['file_groups'][1]['d_headline'] is None
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
content = dict(search_str='foo', headline=True, deep=True)
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
2023-04-12 19:07:16 -06:00
assert response.status_code == HTTPStatus.OK
# FTS headlines use <b>...</b> to highlight matching words.
2023-04-12 19:07:16 -06:00
assert response.json['file_groups'][0]['d_headline'] == '<b>foo</b> baz qux qux'
assert response.json['file_groups'][1]['d_headline'] == '<b>foo</b> bar qux'
@pytest.mark.asyncio
async def test_search_offset(test_session, archive_factory, async_client):
"""Archive search can be offset."""
for i in range(500):
archive_factory('example.com', f'https://example.com/{i}', contents='foo bar')
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
test_session.commit()
data = {'search_str': None, 'offset': 0}
await check_results(async_client, data, list(range(500, 480, -1)))
data = {'search_str': None, 'offset': 20}
await check_results(async_client, data, list(range(480, 460, -1)))
data = {'search_str': None, 'offset': 100}
await check_results(async_client, data, list(range(400, 380, -1)))
data = {'search_str': None, 'offset': 200}
await check_results(async_client, data, list(range(300, 280, -1)))
data = {'search_str': None, 'offset': 500}
await check_results(async_client, data, [])
# Past the last page still reports the true total (a separate COUNT, not COUNT(*) OVER()).
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK
assert response.json['totals']['file_groups'] == 500
2021-12-31 22:35:23 -07:00
@pytest.mark.asyncio
async def test_archives_search_no_query(test_session, archive_factory, async_client):
"""Archive Search API endpoint does not require data in the body."""
2021-12-31 22:35:23 -07:00
# Add 100 random archives.
for _ in range(100):
archive_factory('example.com')
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
test_session.commit()
2021-12-31 22:35:23 -07:00
# All archives are returned when no `search_str` is passed.
request, response = await async_client.post('/api/archive/search', content='{}')
2021-12-31 22:35:23 -07:00
assert response.status_code == HTTPStatus.OK, response.json
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
assert [i['id'] for i in response.json['file_groups']] == list(range(100, 80, -1))
assert response.json['totals']['file_groups'] == 100
2021-12-31 22:35:23 -07:00
# All archives are from "example.com".
data = dict(domain='example.com')
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
2021-12-31 22:35:23 -07:00
assert response.status_code == HTTPStatus.OK, response.json
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
assert [i['id'] for i in response.json['file_groups']] == list(range(100, 80, -1))
assert response.json['totals']['file_groups'] == 100
2021-12-31 22:35:23 -07:00
# No archives are from "example.org".
data = dict(domain='example.org')
request, response = await async_client.post('/api/archive/search', content=json.dumps(data))
2021-12-31 22:35:23 -07:00
assert response.status_code == HTTPStatus.OK, response.json
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
assert [i['id'] for i in response.json['file_groups']] == []
assert response.json['totals']['file_groups'] == 0
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
@skip_macos
@skip_circleci
@pytest.mark.asyncio
async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form,
await_switches, events_fixture):
"""Test converting a SingleFile from the SingleFile browser extension to an Archive."""
singlefile_contents = singlefile_contents_factory(title='upload title', url='https://example.com/single-file-url')
forms = [
dict(name='url', value='https://example.com/form-url'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents')
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
archive, = test_session.query(Archive).all()
assert archive.singlefile_path.is_file(), 'Singlefile should have been saved.'
assert archive.readability_path.is_file(), 'Readability should be extracted.'
assert archive.readability_json_path.is_file(), 'Readability json should be extracted.'
assert archive.screenshot_path.is_file(), 'Screenshot should have been created'
assert archive.file_group.title == 'upload title', 'Title was not correct'
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
events_fixture.assert_has_event('upload_archive')
Creating browser extension to facilitate starting downloads. Chrome/Brave and Firefox supported. The extension is hosted in a sibling repo and is distributed off-store. Each WROLPi release ships the prebuilt artifacts under wrolpi/static/extensions/ and serves them at /api/extensions/<file>: - wrolpi-chrome.zip Load-Unpacked sideload for Chrome / Brave / Edge - wrolpi-firefox.xpi Mozilla-signed (unlisted self-distribution), one-click install in any Firefox - versions.json Per-browser version metadata for the install page Backend (modules/archive/api.py, wrolpi/root_api.py): - POST /api/archive/upload accepts an optional repeated `tagNames` multipart form field; the upload switch handler applies tags to the new FileGroup via FileGroup.set_tags(). Unknown tag names are auto- created inside a per-name SAVEPOINT so concurrent inserts on the same novel name don't abort the transaction (Tag.name has unique=True). - GET /api/extensions returns metadata (per-file availability + size + versions.json contents) for the install page. - GET /api/extensions/<filename> serves the binary, whitelisted by filename. Firefox .xpi is served as application/x-xpinstall WITHOUT a Content-Disposition: attachment header, so Firefox triggers its add-on install handler instead of saving the file. Frontend (app/src/): - New /admin/extension page (linked from AdminRoute) detects the user's browser via navigator.userAgent and renders an install card per Firefox / Chromium / Other. Pulls metadata from /api/extensions to show file size and version, and gracefully reports 'Not yet available' when binaries are missing. - Dashboard's Getters component supports a deep link from the extension's 'Configure Download' button: /?downloader=<archive|video|...>&download_url=<u1>&download_url=<u2> On mount, the matching downloader form auto-opens with URLs pre- filled. Query params are cleared on close so a refresh doesn't re-open the modal. - DownloadMenu and the per-downloader forms accept an `initialUrls` seed via the existing `download` prop pattern, joined with newlines for the `urls` textarea (and first URL for single-URL forms). Behavior is fully backwards compatible: pre-existing API callers can still POST to /api/archive/upload without `tagNames`, and the DashboardPage modal continues to open via the existing button click when no deep-link query params are present.
2026-05-02 14:31:54 -06:00
@pytest.mark.asyncio
async def test_archive_upload_with_tags(test_session, async_client, singlefile_contents_factory, make_multipart_form,
await_switches, tag_factory, archive_factory):
"""Uploading with tagNames applies existing tags and auto-creates new ones.
This test mocks `singlefile_to_archive` so it can run on every platform
(the full archive pipeline depends on a readability/screenshot service
that is not available on macOS). The mocked Archive stands in for what
the real pipeline would have produced; the assertion verifies that the
switch handler applies tags to whichever FileGroup the pipeline returns.
"""
existing = await tag_factory('preexisting')
archive = archive_factory('example.com', 'https://example.com/tagged', 'tagged')
test_session.commit()
async def fake_singlefile_to_archive(singlefile, destination=None):
return archive
singlefile_contents = singlefile_contents_factory(title='tagged', url='https://example.com/tagged')
forms = [
dict(name='url', value='https://example.com/tagged'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents'),
dict(name='tagNames', value=existing.name),
dict(name='tagNames', value='brand-new-tag'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
with patch('modules.archive.lib.singlefile_to_archive', side_effect=fake_singlefile_to_archive):
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
await await_switches()
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert sorted(archive.file_group.tag_names) == ['brand-new-tag', 'preexisting']
@pytest.mark.asyncio
async def test_archive_upload_without_tags_unchanged(test_session, async_client, singlefile_contents_factory,
make_multipart_form, await_switches, archive_factory):
"""Uploads from clients that don't send tagNames (e.g. upstream SingleFile) still work."""
archive = archive_factory('example.com', 'https://example.com/no-tags', 'no-tags')
test_session.commit()
async def fake_singlefile_to_archive(singlefile, destination=None):
return archive
singlefile_contents = singlefile_contents_factory(title='no-tags', url='https://example.com/no-tags')
forms = [
dict(name='url', value='https://example.com/no-tags'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
with patch('modules.archive.lib.singlefile_to_archive', side_effect=fake_singlefile_to_archive):
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
await await_switches()
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert archive.file_group.tag_names == []
@pytest.mark.asyncio
async def test_archive_upload_file_tracking(test_session, async_client, archive_directory, archive_factory,
await_switches, make_multipart_form, image_bytes_factory):
"""Test that uploading an info.json and image file via /api/files/upload properly tracks them in Archive and FileGroup."""
archive = archive_factory('example.com', 'https://example.com/test-url', 'tracking test', screenshot=False)
test_session.commit()
# Verify Archive and FileGroup are created, but screenshot does not exist.
file_group = archive.file_group
assert not archive.screenshot_path, 'Screenshot should not exist yet.'
assert file_group.files is not None, 'FileGroup should have files array'
initial_file_count = len(file_group.files)
# Upload info.json via /api/files/upload
# Extract the prefix from singlefile path (e.g., "2000-01-01-00-00-01_tracking test")
singlefile_stem = archive.singlefile_path.stem # e.g., "2000-01-01-00-00-01_tracking test"
# Create info.json content
info_json_data = {
'title': 'test',
'url': 'https://example.com/test-url',
'custom_field': 'value'
}
info_json_bytes = json.dumps(info_json_data).encode()
# Get relative destination directory
destination = str(get_relative_to_media_directory(archive.singlefile_path.parent))
filename = f'{singlefile_stem}.info.json'
# POST to /api/files/upload
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(info_json_bytes))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=info_json_bytes, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
# Step 4: Verify info.json is tracked
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Assert info.json properties exist
assert archive.info_json_path is not None, 'Archive.info_json_path should exist'
assert archive.info_json_file is not None, 'Archive.info_json_file should exist'
# Assert FileGroup has the correct number of files.
assert len(archive.file_group.files) == initial_file_count + 1, \
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
assert archive.file_group.data.get('info_json_path'), \
'Archive readability info json should be in FileGroup.data'
# Upload an image file and verify it's added to the FileGroup.
# Store file count before adding image
file_count_before_image = len(archive.file_group.files)
# Create image using factory
image_data = image_bytes_factory()
# Upload image with same stem as the Archive (using .jpg to avoid conflict with factory-created .png)
image_filename = f'{singlefile_stem}.jpg'
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=image_filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(image_data))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=image_data, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Image upload failed with status {response.status_code}: {response.json}'
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Assert screenshot properties exist
assert archive.screenshot_path is not None, 'Archive.screenshot_path should exist'
assert archive.screenshot_file is not None, 'Archive.screenshot_file should exist'
# data['screenshot_path'] stores just the filename, screenshot_path resolves to absolute
assert archive.file_group.data.get(
'screenshot_path') == archive.screenshot_path.name, 'Archive screenshot should be in FileGroup.data'
# Assert image is in FileGroup.files (use my_files() to get resolved absolute paths)
file_paths = [f['path'] for f in archive.file_group.my_files()]
assert archive.screenshot_path in file_paths, 'Screenshot should be in FileGroup.files'
# Assert FileGroup has correct number of files
assert len(archive.file_group.files) == file_count_before_image + 1, \
f'FileGroup should have {file_count_before_image + 1} files after adding image'
@pytest.mark.asyncio
async def test_archive_generate_screenshot(test_session, async_client, archive_factory, await_switches,
image_bytes_factory, wrol_mode_fixture):
"""Test generating a screenshot for an Archive that doesn't have one."""
# Mock html_screenshot to avoid Selenium dependency
mock_screenshot_bytes = image_bytes_factory()
with patch('modules.archive.lib.html_screenshot', return_value=mock_screenshot_bytes):
# Test success case: Archive without screenshot
archive = archive_factory('example.com', 'https://example.com/test', 'Test Archive', screenshot=False)
test_session.commit()
# Verify archive has no screenshot
assert archive.screenshot_path is None, 'Archive should not have a screenshot yet'
assert archive.singlefile_path is not None, 'Archive should have a singlefile'
# Request screenshot generation
request, response = await async_client.post(f'/api/archive/{archive.file_group_id}/generate_screenshot')
assert response.status_code == HTTPStatus.OK
assert response.json['message'] == 'Screenshot generation queued'
# Verify screenshot was generated
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert archive.screenshot_path is not None, 'Screenshot should have been generated'
assert archive.screenshot_path.is_file(), 'Screenshot file should exist'
assert archive.screenshot_file is not None, 'Screenshot file should be tracked'
# Test error case: Archive not found
request, response = await async_client.post('/api/archive/99999/generate_screenshot')
assert response.status_code == HTTPStatus.NOT_FOUND
assert 'not found' in response.json['error'].lower()
# Test success case: Archive already has screenshot - should return OK and ensure tracking
original_screenshot_path = archive.screenshot_path
assert original_screenshot_path is not None, 'Archive should have a screenshot'
assert original_screenshot_path.is_file(), 'Screenshot file should exist'
# Note: Previously this test simulated a bug where data['screenshot_path'] could be missing
# while the file existed. This scenario is now auto-fixed by Archive.validate() which calls
# rebuild_data(), so the simulation is no longer necessary.
request, response = await async_client.post(f'/api/archive/{archive.file_group_id}/generate_screenshot')
assert response.status_code == HTTPStatus.OK
assert response.json['message'] == 'Screenshot generation queued'
# Verify screenshot is still there and properly tracked
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert archive.screenshot_path == original_screenshot_path, 'Screenshot path should be unchanged'
assert archive.screenshot_path.is_file(), 'Screenshot file should still exist'
assert archive.screenshot_file is not None, 'Screenshot file should still be tracked'
# IMPORTANT: Also verify FileGroup.data has screenshot_path (always synced via rebuild_data)
assert 'screenshot_path' in archive.file_group.data, \
'screenshot_path should be in FileGroup.data'
assert archive.file_group.data['screenshot_path'] == original_screenshot_path.name, \
'screenshot_path in data should be filename only'
# Test error case: Archive has no singlefile
archive_no_singlefile = archive_factory('example.com', 'https://example.com/no-singlefile', screenshot=False)
test_session.commit()
# Delete the singlefile to simulate missing file
if archive_no_singlefile.singlefile_path and archive_no_singlefile.singlefile_path.is_file():
archive_no_singlefile.singlefile_path.unlink()
request, response = await async_client.post(f'/api/archive/{archive_no_singlefile.file_group_id}/generate_screenshot')
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'no singlefile' in response.json['error'].lower()
# Test WROL mode case
await wrol_mode_fixture(True)
archive_wrol = archive_factory('example.com', 'https://example.com/wrol-test', screenshot=False)
test_session.commit()
request, response = await async_client.post(f'/api/archive/{archive_wrol.file_group_id}/generate_screenshot')
assert response.status_code == HTTPStatus.FORBIDDEN
await wrol_mode_fixture(False)
@pytest.mark.asyncio
async def test_archive_file_format_preview(async_client):
"""Test the archive file format preview endpoint."""
# Test valid format with download_datetime (default format)
data = {'archive_file_format': '%(download_datetime)s_%(title)s.%(ext)s'}
request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK
assert 'preview' in response.json
assert response.json['preview'].endswith('.html')
assert 'Example Page Title' in response.json['preview']
# Verify datetime format YYYY-MM-DD-HH-MM-SS
import re
assert re.search(r'\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}', response.json['preview'])
# Test valid format with download_date (date only)
data = {'archive_file_format': '%(download_date)s_%(title)s.%(ext)s'}
request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK
assert re.search(r'\d{4}-\d{2}-\d{2}_', response.json['preview'])
# Test format with year subdirectory
data = {'archive_file_format': '%(download_year)s/%(download_datetime)s_%(title)s.%(ext)s'}
request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK
assert '/' in response.json['preview'] # Contains subdirectory
# Test invalid format - missing .%(ext)s
data = {'archive_file_format': '%(download_date)s_%(title)s'}
request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'error' in response.json
# Test invalid format - unknown variable
data = {'archive_file_format': '%(invalid_var)s_%(title)s.%(ext)s'}
request, response = await async_client.post('/api/archive/file_format', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'error' in response.json
@pytest.mark.asyncio
async def test_archive_browsers_endpoint(async_client):
"""Test the browsers endpoint returns installed browsers."""
# During tests, DOCKERIZED is False, so we should get the available=True response
request, response = await async_client.get('/api/archive/browsers')
assert response.status_code == HTTPStatus.OK
assert 'browsers' in response.json
assert 'available' in response.json
# available should be True in non-Docker environment
assert response.json['available'] is True
# browsers is a list
assert isinstance(response.json['browsers'], list)
@pytest.mark.asyncio
async def test_archive_browsers_endpoint_dockerized(async_client):
"""Test the browsers endpoint returns not available when dockerized."""
with patch('modules.archive.api.DOCKERIZED', True):
request, response = await async_client.get('/api/archive/browsers')
assert response.status_code == HTTPStatus.OK
assert response.json['browsers'] == []
assert response.json['available'] is False
assert 'message' in response.json
@pytest.mark.asyncio
async def test_delete_archive_untagged(async_client, test_session, archive_factory):
2026-05-03 18:50:22 -06:00
"""An untagged Archive can be deleted via the unified delete endpoint without force."""
archive = archive_factory('example.com', 'https://example.com/1')
test_session.commit()
paths = list(archive.my_paths())
assert paths and all(p.is_file() for p in paths)
2026-05-03 18:50:22 -06:00
body = json.dumps({'file_group_ids': [archive.file_group_id]})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.NO_CONTENT
assert all(not p.exists() for p in paths)
assert test_session.query(Archive).count() == 0
@pytest.mark.asyncio
async def test_delete_archive_tagged_requires_force(async_client, test_session, archive_factory, tag_factory,
await_switches):
"""A tagged Archive returns 409 with FILE_GROUP_IS_TAGGED until force=true is set."""
tag = await tag_factory()
archive = archive_factory('example.com', 'https://example.com/1', tag_names=[tag.name])
test_session.commit()
paths = list(archive.my_paths())
2026-05-03 18:50:22 -06:00
body = json.dumps({'file_group_ids': [archive.file_group_id]})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.CONFLICT
assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
assert len(response.json['file_groups']) == 1
returned = response.json['file_groups'][0]
assert returned['id'] == archive.file_group_id
assert tag.name in returned['tags']
# Nothing was deleted.
assert all(p.is_file() for p in paths)
assert test_session.query(Archive).count() == 1
# With force=true, the Archive is deleted.
2026-05-03 18:50:22 -06:00
body = json.dumps({'file_group_ids': [archive.file_group_id], 'force': True})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.NO_CONTENT
assert all(not p.exists() for p in paths)
assert test_session.query(Archive).count() == 0
await await_switches()
@pytest.mark.asyncio
async def test_delete_archive_atomic_when_some_tagged(async_client, test_session, archive_factory, tag_factory):
"""When deleting multiple Archives and any are tagged, NONE are deleted until force=true."""
tag = await tag_factory()
untagged = archive_factory('example.com', 'https://example.com/1')
tagged = archive_factory('example.com', 'https://example.com/2', tag_names=[tag.name])
test_session.commit()
untagged_paths = list(untagged.my_paths())
tagged_paths = list(tagged.my_paths())
2026-05-03 18:50:22 -06:00
body = json.dumps({'file_group_ids': [untagged.file_group_id, tagged.file_group_id]})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.CONFLICT
assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
# Only the tagged FileGroup is in the response.
returned_ids = {fg['id'] for fg in response.json['file_groups']}
assert returned_ids == {tagged.file_group_id}
# Nothing was deleted.
assert all(p.is_file() for p in untagged_paths)
assert all(p.is_file() for p in tagged_paths)
assert test_session.query(Archive).count() == 2