From 3843279746740f3b6031dccae5554a0c0b2dcf0e Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 14 Apr 2025 21:58:30 -0600 Subject: [PATCH] Fixing the rest of the tests for MacOS. --- modules/archive/test/test_api.py | 3 ++- wrolpi/common.py | 2 +- wrolpi/conftest.py | 2 +- wrolpi/files/api.py | 12 +++++----- wrolpi/files/downloader.py | 3 --- wrolpi/files/lib.py | 38 ++++++++++++++++++++------------ wrolpi/files/models.py | 35 ++++++++++++++++++++--------- wrolpi/files/test/test_api.py | 2 +- wrolpi/files/test/test_lib.py | 19 ++++++++++++---- wrolpi/test/common.py | 1 + wrolpi/test/test_cmd.py | 2 +- wrolpi/test/test_common.py | 1 + wrolpi/test/test_root_api.py | 3 ++- 13 files changed, 80 insertions(+), 43 deletions(-) diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index d865d99b..2dac5f84 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -4,7 +4,7 @@ from http import HTTPStatus import pytest from modules.archive import lib, Archive -from wrolpi.test.common import skip_circleci +from wrolpi.test.common import skip_circleci, skip_macos def check_results(test_client, data, ids): @@ -230,6 +230,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory): assert response.json['domains'] == [] +@skip_macos @skip_circleci @pytest.mark.asyncio async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form, diff --git a/wrolpi/common.py b/wrolpi/common.py index 09d0d08e..87b2672a 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -421,7 +421,7 @@ def get_media_directory() -> Path: if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path): if not is_tempfile(TEST_MEDIA_DIRECTORY): raise RuntimeError(f'Refusing to run test outside tmp directory: {TEST_MEDIA_DIRECTORY}') - return TEST_MEDIA_DIRECTORY + return TEST_MEDIA_DIRECTORY.resolve() return MEDIA_DIRECTORY diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 0c54042a..fe57acb0 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -121,7 +121,7 @@ def test_directory() -> pathlib.Path: Overwrite the media directory with a temporary directory. """ with tempfile.TemporaryDirectory() as tmp_dir: - tmp_path = pathlib.Path(tmp_dir) + tmp_path = pathlib.Path(tmp_dir).resolve() tmp_path.chmod(0o40755) assert tmp_path.is_dir() set_test_media_directory(tmp_path) diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 2d0b4f44..d83bb322 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -1,3 +1,4 @@ +import os.path import pathlib from http import HTTPStatus @@ -7,7 +8,7 @@ from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory, logger, \ - background_task, walk, timer, TRACE_LEVEL + background_task, walk, timer, TRACE_LEVEL, unique_by_predicate from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict from . import lib, schema from ..api_utils import json_response, api_app @@ -46,7 +47,6 @@ async def get_file(_: Request, body: schema.FileRequest): except FileNotFoundError: raise InvalidFile() - # TODO ensure that this is being called during file previews and preview on page load background_task(lib.set_file_viewed(get_media_directory() / body.file)) return json_response({'file': file}) @@ -148,8 +148,7 @@ async def post_search_directories(_, body: schema.DirectoriesSearchRequest): logger.trace(f'post_search_directories: {path=}') try: - matching_directories = lib.get_matching_directories(str(path)) - matching_directories = list(map(pathlib.Path, matching_directories)) + matching_directories = lib.get_matching_directories(path) except FileNotFoundError: matching_directories = [] if logger.isEnabledFor(TRACE_LEVEL): @@ -177,7 +176,10 @@ async def post_search_directories(_, body: schema.DirectoriesSearchRequest): # Return only the top 20 directories. directories = [i.__json__() for i in directories] + for directory in directories: + directory['path'] = get_relative_to_media_directory(directory['path']) directories.extend([{'path': i, 'name': i.name} for i in matching_directories]) + directories = unique_by_predicate(directories, lambda i: i['path']) directories = list(sorted(directories, key=lambda i: i['path']))[:20] if logger.isEnabledFor(TRACE_LEVEL): @@ -185,7 +187,7 @@ async def post_search_directories(_, body: schema.DirectoriesSearchRequest): logger.trace(f'post_search_directories: {directories=}') body = { - 'is_dir': path.is_dir(), + 'is_dir': path.is_dir() and lib.get_real_path_name(path) == path, 'directories': directories, 'channel_directories': channel_directories, 'domain_directories': domain_directories diff --git a/wrolpi/files/downloader.py b/wrolpi/files/downloader.py index 9c18fd19..9138e675 100644 --- a/wrolpi/files/downloader.py +++ b/wrolpi/files/downloader.py @@ -2,7 +2,6 @@ import pathlib import traceback from abc import ABC -from wrolpi.cmd import which from wrolpi.common import get_media_directory, logger, TRACE_LEVEL from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError, IgnoredDirectoryError @@ -13,8 +12,6 @@ __all__ = ['FileDownloader', 'file_downloader'] logger = logger.getChild(__name__) -ARIA2C_PATH = which('aria2c', '/usr/bin/aria2c') - class FileDownloader(Downloader, ABC): """A Downloader which can download an arbitrary HTTP file. diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 6fee96b2..a2e4bdcf 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -37,7 +37,7 @@ from wrolpi.events import Events from wrolpi.files.models import FileGroup, Directory from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config -from wrolpi.vars import PYTEST +from wrolpi.vars import PYTEST, IS_MACOS try: import magic @@ -936,29 +936,30 @@ def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]: return paths -def get_matching_directories(path: Union[str, Path]) -> List[str]: +def get_matching_directories(path: Union[str, Path]) -> List[pathlib.Path]: """ - Return a list of directory strings that start with the provided path. If the path is a directory, return it's + Return a list of directory strings that start with the provided path. If the path is a directory, return its subdirectories, if the directory contains no subdirectories, return the directory. """ - path = str(path) + path: pathlib.Path = pathlib.Path(path) if isinstance(path, str) else path + path = path.resolve() - ignored_directories = {} + ignored_directories = {pathlib.Path(i).resolve() for i in get_wrolpi_config().ignored_directories} - if os.path.isdir(path): + if path.is_dir(): # The provided path is a directory, return its subdirectories, or itself if no subdirectories exist - paths = [os.path.join(path, i) for i in os.listdir(path)] - paths = sorted(i for i in paths if os.path.isdir(i) and i not in ignored_directories) + paths = list(path.iterdir()) + paths = sorted(i for i in paths if i.is_dir() and i not in ignored_directories) if len(paths) == 0: - return [path] + return [get_real_path_name(path.resolve())] return paths - head, tail = os.path.split(path) - paths = os.listdir(head) - paths = [os.path.join(head, i) for i in paths] - pattern = path.lower() + paths = path.parent.iterdir() + prefix = path.name.lower() paths = sorted( - i for i in paths if os.path.isdir(i) and i.lower().startswith(pattern) and i not in ignored_directories) + i for i in paths if i.is_dir() and i.name.lower().startswith(prefix) and i.resolve() not in ignored_directories + ) + paths = unique_by_predicate(paths, None) return paths @@ -1641,3 +1642,12 @@ def get_file_location_href(file: pathlib.Path) -> str: else: query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview))) return f'/files?{query}' + + +def get_real_path_name(path: pathlib.Path) -> pathlib.Path: + """Return the real path name of a file. This is used to get the real path of a file on macOS.""" + if IS_MACOS: + for path_ in path.parent.iterdir(): + if path_.name.lower() == path.name.lower(): + return path_ + return path diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 812e8ae0..8fd10f0e 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -108,7 +108,7 @@ class FileGroup(ModelHelper, Base): def __json__(self) -> dict: from wrolpi.files.lib import split_path_stem_and_suffix _, suffix = split_path_stem_and_suffix(self.primary_path) - + # Get tag names with error handling tag_names = [] for tag_file in self.tag_files: @@ -116,6 +116,8 @@ class FileGroup(ModelHelper, Base): if tag_file.tag is not None: tag_names.append(tag_file.tag.name) except AttributeError: + if PYTEST: + raise # Log the error but continue processing other tags logger.error(f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}") tags = sorted(tag_names) @@ -151,8 +153,13 @@ class FileGroup(ModelHelper, Base): return self.title return self.primary_path.name - def set_viewed(self): - self.viewed = now() + def set_viewed(self, viewed: datetime = None) -> datetime: + """ + :param viewed: Used only for testing! + :return: The datetime that was set. + """ + self.viewed = viewed or now() + return self.viewed @optional_session def set_tags(self, tag_names_or_ids: Iterable[str | int], session: Session = None): @@ -311,14 +318,10 @@ class FileGroup(ModelHelper, Base): @classmethod def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup': """Create a new FileGroup which contains the provided file paths.""" - from wrolpi.files.lib import get_primary_file, get_mimetype, get_unique_files_by_stem - - unique_stems = get_unique_files_by_stem(paths) - if len(unique_stems) > 1: - raise RuntimeError(f'Refusing to create FileGroup with paths that do not share a stem.') + from wrolpi.files.lib import get_primary_file, get_mimetype existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all() - logger.debug(f'FileGroup.from_paths: {len(existing_groups)}') + logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}') if len(existing_groups) == 0: # These paths have not been used previously, create a new FileGroup. file_group = FileGroup() @@ -346,7 +349,7 @@ class FileGroup(ModelHelper, Base): file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths)) file_group.size = sum(i.stat().st_size for i in paths) file_group.mimetype = get_mimetype(file_group.primary_path) - logger.debug(f'FileGroup.from_paths: {file_group}') + logger.trace(f'FileGroup.from_paths: {file_group}') return file_group @@ -363,6 +366,14 @@ class FileGroup(ModelHelper, Base): file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none() return file_group + @staticmethod + @optional_session + def find_by_path(path, session) -> 'FileGroup': + file_group = FileGroup.get_by_path(path, session) + if not file_group: + raise UnknownFile(f'Unable to find FileGroup with path {path}') + return file_group + @property def tag_names(self) -> List[str]: result = [] @@ -371,6 +382,8 @@ class FileGroup(ModelHelper, Base): if tag_file.tag is not None: result.append(tag_file.tag.name) except AttributeError: + if PYTEST: + raise # Log the error but continue processing other tags logger.error(f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}") return result @@ -465,7 +478,7 @@ class FileGroup(ModelHelper, Base): return EBook elif Archive.can_model(self): return Archive - elif Zim.can.model(self): + elif Zim.can_model(self): return Zim def do_model(self, session: Session) -> Optional[ModelHelper]: diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index dcc587c3..fc0e2fb0 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -893,7 +893,7 @@ async def test_delete_directory_recursive(test_session, test_directory, make_fil @pytest.mark.asyncio -async def test_get_file(test_session, async_client, test_directory, make_files_structure, await_background_tasks): +async def test_get_file(test_session, async_client, test_directory, make_files_structure, await_background_tasks, await_switches): """Can get info about a single file.""" make_files_structure({'foo/bar.txt': 'foo contents'}) await lib.refresh_files() diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index e07c5475..32fd6a22 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -22,6 +22,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPr from wrolpi.files import lib, indexers from wrolpi.files.models import FileGroup from wrolpi.tags import TagFile +from wrolpi.test.common import only_macos from wrolpi.vars import PROJECT_DIR, IS_MACOS @@ -603,19 +604,19 @@ def test_matching_directories(make_files_structure, test_directory): # Get all directories starting with f matches = lib.get_matching_directories(test_directory / 'f') - assert matches == [str(test_directory / 'foo')] + assert matches == [test_directory / 'foo'] # Get all directories starting with b, ignore case matches = lib.get_matching_directories(test_directory / 'b') - assert matches == [str(test_directory / 'Bar'), str(test_directory / 'baz')] + assert matches == [test_directory / 'Bar', test_directory / 'baz'] # baz matches, but it has no subdirectories matches = lib.get_matching_directories(test_directory / 'baz') - assert matches == [str(test_directory / 'baz')] + assert matches == [test_directory / 'baz'] # foo is an exact match, return subdirectories matches = lib.get_matching_directories(test_directory / 'foo') - assert matches == [str(test_directory / 'foo/qux')] + assert matches == [test_directory / 'foo/qux'] def test_get_mimetype(example_epub, example_mobi, example_pdf, image_file, video_file): @@ -1158,3 +1159,13 @@ async def test_move_many_files(async_client, test_session, test_directory, make_ assert bar.is_dir() assert (bar / 'foo').is_dir() assert (bar / 'foo/0.txt').is_file() + + +@only_macos +def test_get_real_path_name(test_directory): + (test_directory / 'foo.txt').touch() + assert (test_directory / 'foo.txt').exists() + assert lib.get_real_path_name(test_directory / 'foo.txt') == test_directory / 'foo.txt' + + assert (test_directory / 'FOO.TXT').exists() + assert lib.get_real_path_name(test_directory / 'FOO.txt') == test_directory / 'foo.txt' diff --git a/wrolpi/test/common.py b/wrolpi/test/common.py index 538981ae..1e09cf2b 100644 --- a/wrolpi/test/common.py +++ b/wrolpi/test/common.py @@ -20,6 +20,7 @@ TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False) skip_circleci = pytest.mark.skipif(CIRCLECI, reason='This test is not supported in Circle CI') skip_macos = pytest.mark.skipif(IS_MACOS, reason='This test is not supported on MacOS') +only_macos = pytest.mark.skipif(not IS_MACOS, reason='This test is only supported on MacOS') def wrap_test_db(func): diff --git a/wrolpi/test/test_cmd.py b/wrolpi/test/test_cmd.py index 4078d21f..c9b0c568 100644 --- a/wrolpi/test/test_cmd.py +++ b/wrolpi/test/test_cmd.py @@ -57,7 +57,7 @@ async def test_run_command_cancel(test_directory): """Run a command, cancel it and test that it was cancelled.""" task = asyncio.create_task(cmd.run_command(('sleep', '10'))) try: - await asyncio.wait_for(task, timeout=1) + await asyncio.wait_for(task, timeout=1.1) except TimeoutError: task.cancel() diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 0de84c1b..c9159200 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -884,6 +884,7 @@ async def test_log_level(async_client, test_wrolpi_config): assert api_app.shared_ctx.log_level.value == 10 +@skip_macos @skip_circleci def test_html_screenshot(singlefile_contents_factory): assert common.html_screenshot(singlefile_contents_factory()) diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 6fc4f926..2d47ed00 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -11,7 +11,7 @@ from wrolpi.api_utils import json_error_handler from wrolpi.common import get_wrolpi_config from wrolpi.downloader import Download, get_download_manager_config from wrolpi.errors import ValidationError, SearchEmpty -from wrolpi.test.common import skip_circleci, assert_dict_contains +from wrolpi.test.common import skip_circleci, assert_dict_contains, skip_macos @pytest.mark.asyncio @@ -266,6 +266,7 @@ def test_hotspot_settings(test_session, test_client, test_wrolpi_config): } +@skip_macos @skip_circleci def test_throttle_toggle(test_session, test_client, test_wrolpi_config): get_wrolpi_config().ignored_directories = list()