Fixing the rest of the tests for MacOS.

This commit is contained in:
Roland 2025-04-14 21:58:30 -06:00
parent f264655db7
commit 3843279746
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
13 changed files with 80 additions and 43 deletions

View file

@ -4,7 +4,7 @@ from http import HTTPStatus
import pytest
from modules.archive import lib, Archive
from wrolpi.test.common import skip_circleci
from wrolpi.test.common import skip_circleci, skip_macos
def check_results(test_client, data, ids):
@ -230,6 +230,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
assert response.json['domains'] == []
@skip_macos
@skip_circleci
@pytest.mark.asyncio
async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form,

View file

@ -421,7 +421,7 @@ def get_media_directory() -> Path:
if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path):
if not is_tempfile(TEST_MEDIA_DIRECTORY):
raise RuntimeError(f'Refusing to run test outside tmp directory: {TEST_MEDIA_DIRECTORY}')
return TEST_MEDIA_DIRECTORY
return TEST_MEDIA_DIRECTORY.resolve()
return MEDIA_DIRECTORY

View file

@ -121,7 +121,7 @@ def test_directory() -> pathlib.Path:
Overwrite the media directory with a temporary directory.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = pathlib.Path(tmp_dir)
tmp_path = pathlib.Path(tmp_dir).resolve()
tmp_path.chmod(0o40755)
assert tmp_path.is_dir()
set_test_media_directory(tmp_path)

View file

@ -1,3 +1,4 @@
import os.path
import pathlib
from http import HTTPStatus
@ -7,7 +8,7 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory, logger, \
background_task, walk, timer, TRACE_LEVEL
background_task, walk, timer, TRACE_LEVEL, unique_by_predicate
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
from . import lib, schema
from ..api_utils import json_response, api_app
@ -46,7 +47,6 @@ async def get_file(_: Request, body: schema.FileRequest):
except FileNotFoundError:
raise InvalidFile()
# TODO ensure that this is being called during file previews and preview on page load
background_task(lib.set_file_viewed(get_media_directory() / body.file))
return json_response({'file': file})
@ -148,8 +148,7 @@ async def post_search_directories(_, body: schema.DirectoriesSearchRequest):
logger.trace(f'post_search_directories: {path=}')
try:
matching_directories = lib.get_matching_directories(str(path))
matching_directories = list(map(pathlib.Path, matching_directories))
matching_directories = lib.get_matching_directories(path)
except FileNotFoundError:
matching_directories = []
if logger.isEnabledFor(TRACE_LEVEL):
@ -177,7 +176,10 @@ async def post_search_directories(_, body: schema.DirectoriesSearchRequest):
# Return only the top 20 directories.
directories = [i.__json__() for i in directories]
for directory in directories:
directory['path'] = get_relative_to_media_directory(directory['path'])
directories.extend([{'path': i, 'name': i.name} for i in matching_directories])
directories = unique_by_predicate(directories, lambda i: i['path'])
directories = list(sorted(directories, key=lambda i: i['path']))[:20]
if logger.isEnabledFor(TRACE_LEVEL):
@ -185,7 +187,7 @@ async def post_search_directories(_, body: schema.DirectoriesSearchRequest):
logger.trace(f'post_search_directories: {directories=}')
body = {
'is_dir': path.is_dir(),
'is_dir': path.is_dir() and lib.get_real_path_name(path) == path,
'directories': directories,
'channel_directories': channel_directories,
'domain_directories': domain_directories

View file

@ -2,7 +2,6 @@ import pathlib
import traceback
from abc import ABC
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, logger, TRACE_LEVEL
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError, IgnoredDirectoryError
@ -13,8 +12,6 @@ __all__ = ['FileDownloader', 'file_downloader']
logger = logger.getChild(__name__)
ARIA2C_PATH = which('aria2c', '/usr/bin/aria2c')
class FileDownloader(Downloader, ABC):
"""A Downloader which can download an arbitrary HTTP file.

View file

@ -37,7 +37,7 @@ from wrolpi.events import Events
from wrolpi.files.models import FileGroup, Directory
from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config
from wrolpi.vars import PYTEST
from wrolpi.vars import PYTEST, IS_MACOS
try:
import magic
@ -936,29 +936,30 @@ def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]:
return paths
def get_matching_directories(path: Union[str, Path]) -> List[str]:
def get_matching_directories(path: Union[str, Path]) -> List[pathlib.Path]:
"""
Return a list of directory strings that start with the provided path. If the path is a directory, return it's
Return a list of directory strings that start with the provided path. If the path is a directory, return its
subdirectories, if the directory contains no subdirectories, return the directory.
"""
path = str(path)
path: pathlib.Path = pathlib.Path(path) if isinstance(path, str) else path
path = path.resolve()
ignored_directories = {}
ignored_directories = {pathlib.Path(i).resolve() for i in get_wrolpi_config().ignored_directories}
if os.path.isdir(path):
if path.is_dir():
# The provided path is a directory, return its subdirectories, or itself if no subdirectories exist
paths = [os.path.join(path, i) for i in os.listdir(path)]
paths = sorted(i for i in paths if os.path.isdir(i) and i not in ignored_directories)
paths = list(path.iterdir())
paths = sorted(i for i in paths if i.is_dir() and i not in ignored_directories)
if len(paths) == 0:
return [path]
return [get_real_path_name(path.resolve())]
return paths
head, tail = os.path.split(path)
paths = os.listdir(head)
paths = [os.path.join(head, i) for i in paths]
pattern = path.lower()
paths = path.parent.iterdir()
prefix = path.name.lower()
paths = sorted(
i for i in paths if os.path.isdir(i) and i.lower().startswith(pattern) and i not in ignored_directories)
i for i in paths if i.is_dir() and i.name.lower().startswith(prefix) and i.resolve() not in ignored_directories
)
paths = unique_by_predicate(paths, None)
return paths
@ -1641,3 +1642,12 @@ def get_file_location_href(file: pathlib.Path) -> str:
else:
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
return f'/files?{query}'
def get_real_path_name(path: pathlib.Path) -> pathlib.Path:
"""Return the real path name of a file. This is used to get the real path of a file on macOS."""
if IS_MACOS:
for path_ in path.parent.iterdir():
if path_.name.lower() == path.name.lower():
return path_
return path

View file

@ -108,7 +108,7 @@ class FileGroup(ModelHelper, Base):
def __json__(self) -> dict:
from wrolpi.files.lib import split_path_stem_and_suffix
_, suffix = split_path_stem_and_suffix(self.primary_path)
# Get tag names with error handling
tag_names = []
for tag_file in self.tag_files:
@ -116,6 +116,8 @@ class FileGroup(ModelHelper, Base):
if tag_file.tag is not None:
tag_names.append(tag_file.tag.name)
except AttributeError:
if PYTEST:
raise
# Log the error but continue processing other tags
logger.error(f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
tags = sorted(tag_names)
@ -151,8 +153,13 @@ class FileGroup(ModelHelper, Base):
return self.title
return self.primary_path.name
def set_viewed(self):
self.viewed = now()
def set_viewed(self, viewed: datetime = None) -> datetime:
"""
:param viewed: Used only for testing!
:return: The datetime that was set.
"""
self.viewed = viewed or now()
return self.viewed
@optional_session
def set_tags(self, tag_names_or_ids: Iterable[str | int], session: Session = None):
@ -311,14 +318,10 @@ class FileGroup(ModelHelper, Base):
@classmethod
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
"""Create a new FileGroup which contains the provided file paths."""
from wrolpi.files.lib import get_primary_file, get_mimetype, get_unique_files_by_stem
unique_stems = get_unique_files_by_stem(paths)
if len(unique_stems) > 1:
raise RuntimeError(f'Refusing to create FileGroup with paths that do not share a stem.')
from wrolpi.files.lib import get_primary_file, get_mimetype
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
logger.debug(f'FileGroup.from_paths: {len(existing_groups)}')
logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}')
if len(existing_groups) == 0:
# These paths have not been used previously, create a new FileGroup.
file_group = FileGroup()
@ -346,7 +349,7 @@ class FileGroup(ModelHelper, Base):
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
file_group.size = sum(i.stat().st_size for i in paths)
file_group.mimetype = get_mimetype(file_group.primary_path)
logger.debug(f'FileGroup.from_paths: {file_group}')
logger.trace(f'FileGroup.from_paths: {file_group}')
return file_group
@ -363,6 +366,14 @@ class FileGroup(ModelHelper, Base):
file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none()
return file_group
@staticmethod
@optional_session
def find_by_path(path, session) -> 'FileGroup':
file_group = FileGroup.get_by_path(path, session)
if not file_group:
raise UnknownFile(f'Unable to find FileGroup with path {path}')
return file_group
@property
def tag_names(self) -> List[str]:
result = []
@ -371,6 +382,8 @@ class FileGroup(ModelHelper, Base):
if tag_file.tag is not None:
result.append(tag_file.tag.name)
except AttributeError:
if PYTEST:
raise
# Log the error but continue processing other tags
logger.error(f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
return result
@ -465,7 +478,7 @@ class FileGroup(ModelHelper, Base):
return EBook
elif Archive.can_model(self):
return Archive
elif Zim.can.model(self):
elif Zim.can_model(self):
return Zim
def do_model(self, session: Session) -> Optional[ModelHelper]:

View file

@ -893,7 +893,7 @@ async def test_delete_directory_recursive(test_session, test_directory, make_fil
@pytest.mark.asyncio
async def test_get_file(test_session, async_client, test_directory, make_files_structure, await_background_tasks):
async def test_get_file(test_session, async_client, test_directory, make_files_structure, await_background_tasks, await_switches):
"""Can get info about a single file."""
make_files_structure({'foo/bar.txt': 'foo contents'})
await lib.refresh_files()

View file

@ -22,6 +22,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPr
from wrolpi.files import lib, indexers
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile
from wrolpi.test.common import only_macos
from wrolpi.vars import PROJECT_DIR, IS_MACOS
@ -603,19 +604,19 @@ def test_matching_directories(make_files_structure, test_directory):
# Get all directories starting with f
matches = lib.get_matching_directories(test_directory / 'f')
assert matches == [str(test_directory / 'foo')]
assert matches == [test_directory / 'foo']
# Get all directories starting with b, ignore case
matches = lib.get_matching_directories(test_directory / 'b')
assert matches == [str(test_directory / 'Bar'), str(test_directory / 'baz')]
assert matches == [test_directory / 'Bar', test_directory / 'baz']
# baz matches, but it has no subdirectories
matches = lib.get_matching_directories(test_directory / 'baz')
assert matches == [str(test_directory / 'baz')]
assert matches == [test_directory / 'baz']
# foo is an exact match, return subdirectories
matches = lib.get_matching_directories(test_directory / 'foo')
assert matches == [str(test_directory / 'foo/qux')]
assert matches == [test_directory / 'foo/qux']
def test_get_mimetype(example_epub, example_mobi, example_pdf, image_file, video_file):
@ -1158,3 +1159,13 @@ async def test_move_many_files(async_client, test_session, test_directory, make_
assert bar.is_dir()
assert (bar / 'foo').is_dir()
assert (bar / 'foo/0.txt').is_file()
@only_macos
def test_get_real_path_name(test_directory):
(test_directory / 'foo.txt').touch()
assert (test_directory / 'foo.txt').exists()
assert lib.get_real_path_name(test_directory / 'foo.txt') == test_directory / 'foo.txt'
assert (test_directory / 'FOO.TXT').exists()
assert lib.get_real_path_name(test_directory / 'FOO.txt') == test_directory / 'foo.txt'

View file

@ -20,6 +20,7 @@ TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False)
skip_circleci = pytest.mark.skipif(CIRCLECI, reason='This test is not supported in Circle CI')
skip_macos = pytest.mark.skipif(IS_MACOS, reason='This test is not supported on MacOS')
only_macos = pytest.mark.skipif(not IS_MACOS, reason='This test is only supported on MacOS')
def wrap_test_db(func):

View file

@ -57,7 +57,7 @@ async def test_run_command_cancel(test_directory):
"""Run a command, cancel it and test that it was cancelled."""
task = asyncio.create_task(cmd.run_command(('sleep', '10')))
try:
await asyncio.wait_for(task, timeout=1)
await asyncio.wait_for(task, timeout=1.1)
except TimeoutError:
task.cancel()

View file

@ -884,6 +884,7 @@ async def test_log_level(async_client, test_wrolpi_config):
assert api_app.shared_ctx.log_level.value == 10
@skip_macos
@skip_circleci
def test_html_screenshot(singlefile_contents_factory):
assert common.html_screenshot(singlefile_contents_factory())

View file

@ -11,7 +11,7 @@ from wrolpi.api_utils import json_error_handler
from wrolpi.common import get_wrolpi_config
from wrolpi.downloader import Download, get_download_manager_config
from wrolpi.errors import ValidationError, SearchEmpty
from wrolpi.test.common import skip_circleci, assert_dict_contains
from wrolpi.test.common import skip_circleci, assert_dict_contains, skip_macos
@pytest.mark.asyncio
@ -266,6 +266,7 @@ def test_hotspot_settings(test_session, test_client, test_wrolpi_config):
}
@skip_macos
@skip_circleci
def test_throttle_toggle(test_session, test_client, test_wrolpi_config):
get_wrolpi_config().ignored_directories = list()