mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
A directory rename waits inline on a move job. When that move failed (stale FileGroup rows colliding on primary_path), handle_move reverted the files and sent a failure event but left the job pending. wait_for_job only understands 'complete', so POST /api/files/rename spun until its 300s timeout and the Rename modal never closed. The same sequence could kill file processing for the life of the process. perpetual_signal treated any CancelledError as terminal and logged nothing. Separately, perpetual_tasks_started is a shared Event that survives the worker that set it: docker auto-reloads and worker crashes left every later after_server_start as a no-op. Queued refreshes sat untouched while worker_status reported idle. Restarting the API was the only recovery. Mark failed jobs failed and raise immediately from wait_for_job. Catch unexpected cancellation and reschedule unless the app is actually stopping. Track the perpetual-loop owner PID and retake the claim when that process is gone; every Sanic worker still runs the file-worker pump. Harden transfer_queue against the cross-process Empty race, and stop request processes stealing from the shared public queue.
1635 lines
72 KiB
Python
1635 lines
72 KiB
Python
import hashlib
|
|
import json
|
|
import pathlib
|
|
from datetime import datetime, timezone
|
|
from http import HTTPStatus
|
|
from unittest import mock
|
|
|
|
import pytest
|
|
|
|
from modules.videos import Video
|
|
from wrolpi.common import get_wrolpi_config
|
|
from wrolpi.files.lib import get_mimetype
|
|
from wrolpi.files.models import FileGroup, Directory
|
|
from wrolpi.tags import TagFile
|
|
from wrolpi.test.common import assert_dict_contains
|
|
from wrolpi.vars import PROJECT_DIR
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_files_api(test_session, async_client, make_files_structure, test_directory):
|
|
files = [
|
|
'archives/bar.txt',
|
|
'archives/baz/bar.txt',
|
|
'archives/baz/foo.txt',
|
|
'archives/foo.txt',
|
|
'empty directory/',
|
|
'videos/other video.mp4',
|
|
'videos/some video.mp4',
|
|
'lost+found/', # Should always be ignored.
|
|
]
|
|
files = make_files_structure(files)
|
|
files[0].write_text('bar contents')
|
|
|
|
async def check_get_files(directories, expected_files):
|
|
request, response = await async_client.post('/api/files', content=json.dumps({'directories': directories}))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert not response.json.get('errors')
|
|
# The first dict is the media directory.
|
|
children = response.json['files']
|
|
assert_dict_contains(children, expected_files)
|
|
|
|
# Requesting no directories results in the top-level results.
|
|
expected = {
|
|
'archives/': {'path': 'archives/', 'is_empty': False},
|
|
'empty directory/': {'path': 'empty directory/', 'is_empty': True},
|
|
'videos/': {'path': 'videos/', 'is_empty': False}
|
|
}
|
|
await check_get_files([], expected)
|
|
# empty directory is empty
|
|
expected = {
|
|
'archives/': {'path': 'archives/', 'is_empty': False},
|
|
'empty directory/': {'path': 'empty directory/', 'children': {}, 'is_empty': True},
|
|
'videos/': {'path': 'videos/', 'is_empty': False}
|
|
}
|
|
await check_get_files(['empty directory'], expected)
|
|
|
|
expected = {
|
|
'archives/': {
|
|
'path': 'archives/',
|
|
'children': {
|
|
'foo.txt': {'path': 'archives/foo.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
|
|
'bar.txt': {'path': 'archives/bar.txt', 'size': 12, 'mimetype': 'text/plain'},
|
|
'baz/': {'path': 'archives/baz/', 'is_empty': False},
|
|
},
|
|
'is_empty': False,
|
|
},
|
|
'empty directory/': {'path': 'empty directory/', 'is_empty': True},
|
|
'videos/': {'path': 'videos/', 'is_empty': False}
|
|
}
|
|
await check_get_files(['archives'], expected)
|
|
|
|
# Sub-directories are supported.
|
|
expected = {
|
|
'archives/': {
|
|
'path': 'archives/',
|
|
'children': {
|
|
'foo.txt': {'path': 'archives/foo.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
|
|
'bar.txt': {'path': 'archives/bar.txt', 'size': 12, 'mimetype': 'text/plain'},
|
|
'baz/': {
|
|
'path': 'archives/baz/',
|
|
'children': {
|
|
'bar.txt': {'path': 'archives/baz/bar.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
|
|
'foo.txt': {'path': 'archives/baz/foo.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
|
|
},
|
|
'is_empty': False,
|
|
}
|
|
},
|
|
'is_empty': False
|
|
},
|
|
'empty directory/': {'path': 'empty directory/', 'is_empty': True},
|
|
'videos/': {'path': 'videos/', 'is_empty': False},
|
|
}
|
|
await check_get_files(['archives', 'archives/baz'], expected)
|
|
# Requesting only a subdirectory also returns `archives` contents.
|
|
await check_get_files(['archives/baz'], expected)
|
|
|
|
expected = {
|
|
'archives/': {
|
|
'path': 'archives/',
|
|
'children': {
|
|
'foo.txt': {'path': 'archives/foo.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
|
|
'bar.txt': {'path': 'archives/bar.txt', 'size': 12, 'mimetype': 'text/plain'},
|
|
'baz/': {'path': 'archives/baz/', 'is_empty': False},
|
|
}, 'is_empty': False
|
|
},
|
|
'empty directory/': {'path': 'empty directory/', 'is_empty': True},
|
|
'videos/': {
|
|
'path': 'videos/',
|
|
'children': {
|
|
'other video.mp4': {'path': 'videos/other video.mp4', 'size': 0, 'mimetype': 'inode/x-empty'},
|
|
'some video.mp4': {'path': 'videos/some video.mp4', 'size': 0, 'mimetype': 'inode/x-empty'},
|
|
},
|
|
'is_empty': False
|
|
}
|
|
}
|
|
await check_get_files(['archives', 'videos'], expected)
|
|
# Order does not matter.
|
|
await check_get_files(['videos', 'archives'], expected)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_files_api_hides_ds_store(test_session, async_client, make_files_structure, test_directory):
|
|
"""macOS metadata files (.DS_Store / ._.DS_Store) are never shown in the FileBrowser."""
|
|
make_files_structure([
|
|
'.DS_Store', # Top-level metadata file should be hidden.
|
|
'._.DS_Store', # Top-level AppleDouble metadata file should be hidden.
|
|
'archives/foo.txt',
|
|
'archives/.DS_Store', # Nested metadata file should be hidden.
|
|
'archives/._.DS_Store', # Nested AppleDouble metadata file should be hidden.
|
|
])
|
|
|
|
async def get_files(directories):
|
|
request, response = await async_client.post('/api/files', content=json.dumps({'directories': directories}))
|
|
assert response.status_code == HTTPStatus.OK
|
|
return response.json['files']
|
|
|
|
# Top-level listing hides the metadata files.
|
|
top_level = await get_files([])
|
|
assert '.DS_Store' not in top_level
|
|
assert '._.DS_Store' not in top_level
|
|
assert 'archives/' in top_level
|
|
|
|
# Nested listing hides the metadata files, but still shows real files.
|
|
nested = await get_files(['archives'])
|
|
children = nested['archives/']['children']
|
|
assert '.DS_Store' not in children
|
|
assert '._.DS_Store' not in children
|
|
assert 'foo.txt' in children
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_file(test_session, async_client, make_files_structure, test_directory):
|
|
files = ['bar.txt', 'baz/', 'foo']
|
|
make_files_structure(files)
|
|
|
|
# Delete a file.
|
|
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['bar.txt']}))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert not (test_directory / 'bar.txt').is_file()
|
|
assert (test_directory / 'baz').is_dir()
|
|
|
|
# Delete a directory.
|
|
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['baz']}))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert not (test_directory / 'bar.txt').is_file()
|
|
assert not (test_directory / 'baz').is_dir()
|
|
|
|
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['bad file']}))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
'paths', [
|
|
[],
|
|
['', ],
|
|
]
|
|
)
|
|
async def test_delete_invalid_file(async_client, paths):
|
|
"""Some paths must be passed."""
|
|
with mock.patch('wrolpi.files.api.lib.delete') as mock_delete_file:
|
|
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': paths}))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
mock_delete_file.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_wrol_mode(async_client, wrol_mode_fixture):
|
|
"""Can't delete a file when WROL Mode is enabled."""
|
|
await wrol_mode_fixture(True)
|
|
|
|
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['foo', ]}))
|
|
assert response.status_code == HTTPStatus.FORBIDDEN
|
|
assert response.json['code'] == 'WROL_MODE_ENABLED'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_file_groups_mixed_batch(async_client, test_session, video_factory, archive_factory):
|
|
"""A single POST /api/files/delete_groups can delete a mixture of model types in one call."""
|
|
from modules.archive.models import Archive
|
|
video = video_factory(with_video_file=True)
|
|
archive = archive_factory('example.com', 'https://example.com/1')
|
|
test_session.commit()
|
|
video_paths = list(video.file_group.my_paths())
|
|
archive_paths = list(archive.my_paths())
|
|
|
|
body = json.dumps({'file_group_ids': [video.file_group_id, archive.file_group_id]})
|
|
request, response = await async_client.post('/api/files/delete_groups', content=body)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
assert all(not p.exists() for p in video_paths)
|
|
assert all(not p.exists() for p in archive_paths)
|
|
assert test_session.query(Video).count() == 0
|
|
assert test_session.query(Archive).count() == 0
|
|
assert test_session.query(FileGroup).count() == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_file_groups_default_deleter(async_client, test_session, make_files_structure,
|
|
refresh_files):
|
|
"""A FileGroup with no specialized model (e.g., a plain text file) is removed via FileGroup.delete."""
|
|
foo, = make_files_structure({'foo.txt': 'hello'})
|
|
await refresh_files()
|
|
|
|
fg = test_session.query(FileGroup).one()
|
|
assert fg.model is None # plain text isn't modeled by Video/Archive/Doc/Zim
|
|
fg_id = fg.id
|
|
|
|
body = json.dumps({'file_group_ids': [fg_id]})
|
|
request, response = await async_client.post('/api/files/delete_groups', content=body)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
assert not foo.exists()
|
|
assert test_session.query(FileGroup).filter_by(id=fg_id).one_or_none() is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_file_groups_unknown_ids(async_client, test_session):
|
|
"""Deleting non-existent IDs returns 404 from UnknownFile."""
|
|
body = json.dumps({'file_group_ids': [99999]})
|
|
request, response = await async_client.post('/api/files/delete_groups', content=body)
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
assert response.json['code'] == 'UNKNOWN_FILE'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_file_groups_empty_ids(async_client):
|
|
"""An empty file_group_ids list returns 400 (InvalidFile)."""
|
|
body = json.dumps({'file_group_ids': []})
|
|
request, response = await async_client.post('/api/files/delete_groups', content=body)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_files_search_recent(test_session, test_directory, async_client, video_file_factory, refresh_files):
|
|
"""File search can return the most recently viewed files."""
|
|
video_file_factory(test_directory / 'foo.mp4'), video_file_factory(test_directory / 'bar.mp4')
|
|
baz = (test_directory / 'baz.txt')
|
|
baz.write_text('baz contents')
|
|
await refresh_files()
|
|
|
|
# baz is most recently viewed, foo has not been viewed.
|
|
bar, baz, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
|
bar.set_viewed(datetime(2000, 1, 1, 1, 1, 1, tzinfo=timezone.utc))
|
|
baz.set_viewed(datetime(2000, 1, 1, 1, 1, 2, tzinfo=timezone.utc))
|
|
test_session.commit()
|
|
|
|
# Results match the viewed order. foo is not viewed.
|
|
body = dict(order='viewed')
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_groups'], response.json
|
|
assert [i['name'] for i in response.json['file_groups']] == ['baz.txt', 'bar.mp4']
|
|
|
|
# Results match the reverse viewed order. foo is not viewed.
|
|
body = dict(order='-viewed')
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_groups'], response.json
|
|
assert [i['name'] for i in response.json['file_groups']] == ['bar.mp4', 'baz.txt']
|
|
|
|
# foo is included because `search_str` has a value.
|
|
body = dict(search_str='foo', order='viewed')
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_groups'], response.json
|
|
assert [i['name'] for i in response.json['file_groups']] == ['foo.mp4']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_files_search_null_params(test_session, test_directory, async_client, video_file_factory,
|
|
refresh_files):
|
|
"""Explicit JSON nulls (as the dashboard sends) must not break the search.
|
|
|
|
Regression test: the frontend sends `{"search_str": null, "offset": null, ...}`; explicit
|
|
nulls bypass the schema defaults and reached `OFFSET NULL`, which Postgres treated as 0 but
|
|
SQLite rejects with 'datatype mismatch'."""
|
|
video_file_factory(test_directory / 'foo.mp4')
|
|
await refresh_files()
|
|
file_group = test_session.query(FileGroup).one()
|
|
file_group.set_viewed(datetime(2000, 1, 1, 1, 1, 1, tzinfo=timezone.utc))
|
|
test_session.commit()
|
|
|
|
# The exact dashboard "recent files" payload.
|
|
body = {'search_str': None, 'offset': None, 'limit': 12, 'any_tag': False, 'tag_names': [], 'order': 'viewed'}
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
assert [i['name'] for i in response.json['file_groups']] == ['foo.mp4']
|
|
|
|
# Null limit as well.
|
|
body = {'search_str': None, 'offset': None, 'limit': None}
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
assert response.json['file_groups']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_files_search_stemming(test_session, async_client, make_files_structure, refresh_files,
|
|
assert_files_search):
|
|
"""Searching any grammatical variant of a word finds files named with other variants.
|
|
|
|
End-to-end stemming contract: "watch" must find "watching", "fishes" must find "fishing"."""
|
|
make_files_structure(['watching birds.mp4', 'fishing guide.txt', 'two cats.txt', 'unrelated.txt'])
|
|
await refresh_files()
|
|
|
|
await assert_files_search('watch', [dict(name='watching birds.mp4')])
|
|
await assert_files_search('watched', [dict(name='watching birds.mp4')])
|
|
await assert_files_search('fish', [dict(name='fishing guide.txt')])
|
|
await assert_files_search('fishes', [dict(name='fishing guide.txt')])
|
|
await assert_files_search('cat', [dict(name='two cats.txt')])
|
|
# Stemmers do not lemmatize: derived words are their own terms (same as the old engine).
|
|
await assert_files_search('runner', [])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_files_search(test_session, async_client, make_files_structure, assert_files_search):
|
|
# You can search an empty directory.
|
|
await assert_files_search('nothing', [])
|
|
|
|
# Create files in the temporary directory. Add some contents so the mimetype can be tested.
|
|
files = [
|
|
'foo_is_the_name.txt',
|
|
'archives/bar.txt',
|
|
'baz.mp4',
|
|
'baz baz two.mp4'
|
|
]
|
|
foo, bar, baz, baz2 = make_files_structure(files)
|
|
foo.write_text('foo contents')
|
|
bar.write_text('the bar contents')
|
|
baz.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
|
|
baz2.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
|
|
|
|
# Refresh so files can be searched.
|
|
request, response = await async_client.post('/api/files/refresh')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
await assert_files_search('foo', [dict(primary_path='foo_is_the_name.txt')])
|
|
await assert_files_search('bar', [dict(primary_path='archives/bar.txt')])
|
|
await assert_files_search('baz', [dict(primary_path='baz baz two.mp4'), dict(primary_path='baz.mp4')])
|
|
await assert_files_search('two', [dict(primary_path='baz baz two.mp4')])
|
|
await assert_files_search('nothing', [])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_files_search_deep(test_session, async_client, make_files_structure):
|
|
"""Search defaults to the fast ABC column (title/author/description); `deep` opts into d_text
|
|
(captions/file contents)."""
|
|
guide, lecture = make_files_structure([
|
|
'gardening guide.txt',
|
|
'lecture.txt',
|
|
])
|
|
guide.write_text('planting tips')
|
|
# "gardening" only appears in this file's contents (d_text), not its name.
|
|
lecture.write_text('a lecture about gardening')
|
|
|
|
request, response = await async_client.post('/api/files/refresh')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
async def search(**kwargs):
|
|
request, response = await async_client.post('/api/files/search', json=kwargs)
|
|
assert response.status_code == HTTPStatus.OK
|
|
return response.json['file_groups']
|
|
|
|
# Default (fast) search only matches the title.
|
|
results = await search(search_str='gardening')
|
|
assert [i['primary_path'] for i in results] == ['gardening guide.txt']
|
|
|
|
# Deep search also matches file contents; the title match ranks higher (weight A vs D).
|
|
results = await search(search_str='gardening', deep=True)
|
|
assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
|
|
|
|
# Headlines work in both modes.
|
|
results = await search(search_str='gardening', headline=True)
|
|
assert [i['primary_path'] for i in results] == ['gardening guide.txt']
|
|
assert '<b>gardening</b>' in results[0]['title_headline']
|
|
results = await search(search_str='gardening', deep=True, headline=True)
|
|
assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
|
|
assert '<b>gardening</b>' in results[1]['d_headline']
|
|
|
|
# The estimate endpoint returns both counts: `file_groups` matches the default fast search, and
|
|
# `file_groups_deep` shows what deep search would find, so the UI can offer deep search.
|
|
request, response = await async_client.post('/api/search_file_estimates',
|
|
json=dict(search_str='gardening'))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_groups'] == 1
|
|
assert response.json['file_groups_deep'] == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_files_search_by_suffix(test_session, async_client, make_files_structure):
|
|
"""Files can be filtered by their primary file's suffix (used by the firmware flasher to find .bin files)."""
|
|
files = [
|
|
'software/MALVEKE.ino.bin',
|
|
'software/bffb/esp32_marauder.ino.bootloader.bin',
|
|
'software/firmware-heltec.uf2',
|
|
'software/firmware-r1-neo.uf2',
|
|
'notes.txt',
|
|
]
|
|
created = make_files_structure(files)
|
|
for path in created:
|
|
path.write_bytes(b'\x00\x01\x02\x03')
|
|
|
|
request, response = await async_client.post('/api/files/refresh')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
async def search(**kwargs):
|
|
request, response = await async_client.post('/api/files/search', json=kwargs)
|
|
assert response.status_code == HTTPStatus.OK
|
|
return sorted(fg['primary_path'] for fg in response.json['file_groups'])
|
|
|
|
# Exact suffix match, normalized whether or not a leading dot is given, and case-insensitive.
|
|
assert await search(suffix='.bin') == ['software/MALVEKE.ino.bin',
|
|
'software/bffb/esp32_marauder.ino.bootloader.bin']
|
|
assert await search(suffix='bin') == ['software/MALVEKE.ino.bin',
|
|
'software/bffb/esp32_marauder.ino.bootloader.bin']
|
|
assert await search(suffix='.BIN') == ['software/MALVEKE.ino.bin',
|
|
'software/bffb/esp32_marauder.ino.bootloader.bin']
|
|
assert await search(suffix='.uf2') == ['software/firmware-heltec.uf2', 'software/firmware-r1-neo.uf2']
|
|
assert await search(suffix='.txt') == ['notes.txt']
|
|
# A suffix that matches nothing returns no results.
|
|
assert await search(suffix='.gguf') == []
|
|
|
|
# The `path` filter matches anywhere in the primary_path (case-insensitive), so a directory name works.
|
|
assert await search(path='bffb') == ['software/bffb/esp32_marauder.ino.bootloader.bin']
|
|
assert await search(path='BFFB') == ['software/bffb/esp32_marauder.ino.bootloader.bin']
|
|
assert await search(path='software/') == ['software/MALVEKE.ino.bin',
|
|
'software/bffb/esp32_marauder.ino.bootloader.bin',
|
|
'software/firmware-heltec.uf2', 'software/firmware-r1-neo.uf2']
|
|
# `suffix` and `path` combine (the firmware picker sends both).
|
|
assert await search(suffix='.bin', path='bffb') == ['software/bffb/esp32_marauder.ino.bootloader.bin']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_files_search_attaches_doc_section_hint(async_client, test_session, test_directory,
|
|
example_epub, refresh_files):
|
|
"""The global /api/files/search endpoint attaches `section_hint` to doc-modeled
|
|
results so the UI can deep-link into the matching chapter. Regression test for a
|
|
bug where only the docs-specific endpoint (/api/docs/search) attached hints."""
|
|
await refresh_files()
|
|
|
|
# "chapter" only appears in the EPUB body (d_text), so deep search is required to match it.
|
|
body = dict(search_str='chapter', deep=True)
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
epub_results = [r for r in response.json['file_groups']
|
|
if r.get('model') == 'doc'
|
|
and (r.get('mimetype') or '').startswith('application/epub')]
|
|
assert len(epub_results) == 1, f'Expected 1 EPUB result, got: {response.json}'
|
|
result = epub_results[0]
|
|
assert 'section_hint' in result, \
|
|
f'doc result has no section_hint on global search path: {result}'
|
|
hint = result['section_hint']
|
|
assert hint['kind'] == 'epub_spine'
|
|
# The test EPUB's spine order is [titlepage, start, chapter1]; the chapter1
|
|
# document holds the "Chapter one" body, so its spine ordinal is 2.
|
|
assert hint['ordinal'] == 2
|
|
assert hint['label']
|
|
# Snippet uses our sentinel markers, not raw HTML.
|
|
assert '[[WROLPI_HL]]' in hint['snippet']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory, refresh_files):
|
|
one, two = await tag_factory(), await tag_factory()
|
|
files = [
|
|
'foo.txt',
|
|
'foo bar.txt',
|
|
'bar.txt',
|
|
]
|
|
make_files_structure(files)
|
|
await refresh_files()
|
|
bar, foobar, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
|
assert bar.primary_path.name == 'bar.txt' \
|
|
and foo.primary_path.name == 'foo.txt' \
|
|
and foobar.primary_path.name == 'foo bar.txt'
|
|
foo.add_tag(test_session, one.id)
|
|
foobar.add_tag(test_session, two.id)
|
|
test_session.commit()
|
|
|
|
# Only `foo` is tagged with `one`
|
|
body = dict(search_str='foo', tag_names=['one'])
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt'}
|
|
|
|
# Both `foo bar` and `foo` are tagged.
|
|
body = dict(search_str='foo', any_tag=True)
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt', 'foo bar.txt'}
|
|
|
|
# `bar` is not tagged.
|
|
body = dict(search_str='bar', any_tag=True)
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo bar.txt'}
|
|
|
|
# Cannot search for any_tag, and tag names.
|
|
body = dict(search_str='foo', tag_names=['one'], any_tag=True)
|
|
request, response = await async_client.post('/api/files/search', json=body)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_refresh_files_list(test_session, async_client, make_files_structure, test_directory, video_bytes):
|
|
"""The user can request to refresh specific files."""
|
|
make_files_structure({
|
|
'bar.txt': 'hello',
|
|
'bar.mp4': video_bytes,
|
|
})
|
|
|
|
# Only the single file that was refreshed is discovered.
|
|
content = json.dumps({'paths': ['bar.txt']})
|
|
request, response = await async_client.post('/api/files/refresh', content=content)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert test_session.query(FileGroup).count() == 1
|
|
group: FileGroup = test_session.query(FileGroup).one()
|
|
assert len(group.files) == 1
|
|
|
|
request, response = await async_client.post('/api/files/refresh')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
group: FileGroup = test_session.query(FileGroup).one()
|
|
# Video validation creates .ffprobe.json, so we may have 2 or 3 files
|
|
assert len(group.files) >= 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_file_statistics(test_session, async_client, test_directory, example_pdf, example_mobi, example_epub,
|
|
video_file):
|
|
"""A summary of File statistics can be fetched."""
|
|
# Give each file a unique stem.
|
|
video_file.rename(test_directory / 'video.mp4')
|
|
example_pdf.rename(test_directory / 'pdf.pdf')
|
|
example_mobi.rename(test_directory / 'mobi.mobi')
|
|
example_epub.rename(test_directory / 'epub.epub')
|
|
|
|
# Statistics can be fetched while empty.
|
|
request, response = await async_client.get('/api/statistics')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_statistics'] == {
|
|
'archive_count': 0,
|
|
'audio_count': 0,
|
|
'ebook_count': 0,
|
|
'image_count': 0,
|
|
'pdf_count': 0,
|
|
'tagged_files': 0,
|
|
'tagged_zims': 0,
|
|
'tags_count': 0,
|
|
'total_count': 0,
|
|
'total_size': 0,
|
|
'video_count': 0,
|
|
'zip_count': 0,
|
|
}
|
|
|
|
await async_client.post('/api/files/refresh')
|
|
|
|
request, response = await async_client.get('/api/statistics')
|
|
assert response.status_code == HTTPStatus.OK
|
|
stats = response.json['file_statistics']
|
|
stats.pop('total_size')
|
|
assert stats == {
|
|
'archive_count': 0,
|
|
'audio_count': 0,
|
|
'ebook_count': 2,
|
|
'image_count': 0,
|
|
'pdf_count': 1,
|
|
'tagged_files': 0,
|
|
'tagged_zims': 0,
|
|
'tags_count': 0,
|
|
'total_count': 7, # extracted covers (epub + pdf) + .ffprobe.json
|
|
'video_count': 1,
|
|
'zip_count': 0,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_file_group_tag_by_primary_path(test_session, async_client, test_directory, example_singlefile,
|
|
tag_factory, insert_file_group):
|
|
singlefile = FileGroup.from_paths(test_session, example_singlefile)
|
|
tag1 = await tag_factory()
|
|
tag2 = await tag_factory()
|
|
test_session.commit()
|
|
|
|
# FileGroup can be tagged with its primary_path.
|
|
content = dict(file_group_primary_path=str(singlefile.primary_path.relative_to(test_directory)), tag_name=tag1.name)
|
|
request, response = await async_client.post('/api/files/tag', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
assert test_session.query(TagFile).count() == 1
|
|
|
|
# FileGroup can be tagged with its id.
|
|
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
|
|
request, response = await async_client.post('/api/files/tag', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
assert test_session.query(TagFile).count() == 2
|
|
|
|
# FileGroup can be untagged with its primary_path.
|
|
content = dict(file_group_id=singlefile.id, tag_id=tag1.id)
|
|
request, response = await async_client.post('/api/files/untag', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert test_session.query(TagFile).count() == 1
|
|
|
|
# FileGroup can be untagged with its id.
|
|
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
|
|
request, response = await async_client.post('/api/files/untag', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert test_session.query(TagFile).count() == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_file_group_tag(async_client):
|
|
request, response = await async_client.post('/api/files/tag', content=json.dumps(dict(tag_id=1)))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert 'file_group_id' in response.json['error']
|
|
|
|
request, response = await async_client.post('/api/files/tag', content=json.dumps(dict(file_group_id=1)))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert 'tag_id' in response.json['error']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_directories(async_client, test_session, test_directory, make_files_structure,
|
|
assert_directories, refresh_files):
|
|
"""Directories can be searched by name."""
|
|
make_files_structure(['foo/one.txt', 'foo/two.txt', 'bar/one.txt'])
|
|
await refresh_files()
|
|
assert_directories({'foo', 'bar'})
|
|
# Create directory that was not refreshed.
|
|
(test_directory / 'baz').mkdir()
|
|
|
|
# More than one character required.
|
|
content = dict(path='f')
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK, response.content
|
|
assert response.json['directories'] == []
|
|
assert response.json['is_dir'] is False
|
|
|
|
# Can search using partial directory name.
|
|
content = dict(path='fo')
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert [i['path'] for i in response.json['directories']] == ['foo', ]
|
|
assert response.json['is_dir'] is False
|
|
|
|
# Searching directory exactly.
|
|
content = dict(path='foo')
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert [i['path'] for i in response.json['directories']] == ['foo', ]
|
|
assert response.json['is_dir'] is True
|
|
|
|
# Case is ignored.
|
|
content = dict(path='BAR')
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert [i['path'] for i in response.json['directories']] == ['bar', ]
|
|
assert response.json['is_dir'] is False
|
|
|
|
# Can search directories not yet in DB.
|
|
content = dict(path='ba')
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert [i['path'] for i in response.json['directories']] == ['bar', 'baz']
|
|
assert response.json['is_dir'] is False
|
|
|
|
# Searching for something that does not exist.
|
|
content = dict(path='does not exist')
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert [i['path'] for i in response.json['directories']] == []
|
|
|
|
# Only first 20 "foo" are returned.
|
|
make_files_structure([
|
|
'fooo/',
|
|
'foooo/',
|
|
'fooooo/',
|
|
'foooooo/',
|
|
'fooooooo/',
|
|
'foooooooo/',
|
|
'fooooooooo/',
|
|
'foooooooooo/',
|
|
'fooooooooooo/',
|
|
'foooooooooooo/',
|
|
'fooooooooooooo/',
|
|
'foooooooooooooo/',
|
|
'fooooooooooooooo/',
|
|
'foooooooooooooooo/',
|
|
'fooooooooooooooooo/',
|
|
'foooooooooooooooooo/',
|
|
'fooooooooooooooooooo/',
|
|
'foooooooooooooooooooo/',
|
|
'fooooooooooooooooooooo/',
|
|
'foooooooooooooooooooooo/',
|
|
'fooooooooooooooooooooooo/',
|
|
])
|
|
await async_client.post('/api/files/refresh')
|
|
content = dict(path='fo')
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert [i['path'] for i in response.json['directories']] == [f'f{"o" * i}' for i in range(2, 22)]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_search_directories(test_session, async_client, make_files_structure):
|
|
"""Directory names can be searched. This endpoint also returns Channel and Domain directories."""
|
|
channel1_dir, channel2_dir, domain_dir, _ = make_files_structure([
|
|
'dir1/',
|
|
'dir2/',
|
|
'dir3/',
|
|
'dir4/',
|
|
])
|
|
request, response = await async_client.post('/api/files/refresh')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
from modules.videos.models import Channel
|
|
# Use from_config to create channels properly (creates Collection first)
|
|
channel1 = Channel.from_config(test_session, {'directory': channel1_dir, 'name': 'Channel Name'})
|
|
channel2 = Channel.from_config(test_session, {'directory': channel2_dir, 'name': 'OtherChannel'})
|
|
from wrolpi.collections import Collection
|
|
domain_collection = Collection(directory=domain_dir, name='example.com', kind='domain')
|
|
test_session.add(domain_collection)
|
|
test_session.commit()
|
|
|
|
# All directories contain "di". The names of the Channel and Directory do not match.
|
|
content = {'path': 'di'}
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['directories'] == [{'name': 'dir1', 'path': 'dir1'}, {'name': 'dir2', 'path': 'dir2'},
|
|
{'name': 'dir3', 'path': 'dir3'}, {'name': 'dir4', 'path': 'dir4'}]
|
|
assert response.json['channel_directories'] == []
|
|
assert response.json['domain_directories'] == []
|
|
|
|
# Channel name matches.
|
|
content = {'path': 'Chan'}
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['directories'] == []
|
|
assert response.json['channel_directories'] == [
|
|
{'name': 'Channel Name', 'path': 'dir1'},
|
|
{'name': 'OtherChannel', 'path': 'dir2'},
|
|
]
|
|
assert response.json['domain_directories'] == []
|
|
|
|
# "OtherChannel" matches even though the case is wrong, and it has a space.
|
|
content = {'path': 'other channel'}
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
# Channel name matches.
|
|
assert response.json['directories'] == []
|
|
assert response.json['channel_directories'] == [{'name': 'OtherChannel', 'path': 'dir2'}]
|
|
assert response.json['domain_directories'] == []
|
|
|
|
content = {'path': 'exam'}
|
|
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
# Domain name matches.
|
|
assert response.json['directories'] == []
|
|
assert response.json['channel_directories'] == []
|
|
assert response.json['domain_directories'] == [{'domain': 'example.com', 'path': 'dir3'}]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_upload_directory(test_session, async_client, test_directory, make_files_structure,
|
|
make_multipart_form):
|
|
"""A file can be uploaded in a directory in the destination."""
|
|
make_files_structure(['uploads/'])
|
|
|
|
forms = [
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='filename', value='/foo/bar.txt'), # notice the directory
|
|
dict(name='totalChunks', value='0'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value='3'),
|
|
dict(name='chunk', value='foo', filename='chunk')
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
assert (test_directory / 'uploads/foo/bar.txt').is_file()
|
|
assert (test_directory / 'uploads/foo/bar.txt').read_text() == 'foo'
|
|
|
|
assert test_session.query(FileGroup).count() == 1
|
|
assert test_session.query(Directory).count() == 2
|
|
assert {i for i, in test_session.query(Directory.path)} == {
|
|
test_directory / 'uploads',
|
|
test_directory / 'uploads/foo',
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_upload(test_session, async_client, test_directory, make_files_structure, make_multipart_form,
|
|
tag_factory, video_bytes, test_wrolpi_config):
|
|
"""A file can be uploaded in chunks directly to the destination."""
|
|
# Disable ffprobe json file creation to avoid unexpected files in test assertions
|
|
get_wrolpi_config().save_ffprobe_json = False
|
|
|
|
make_files_structure(['uploads/'])
|
|
tag1, tag2 = await tag_factory(), await tag_factory()
|
|
|
|
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
|
|
forms = [
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='filename', value='video.mp4'),
|
|
dict(name='totalChunks', value='1'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value=1_000_000),
|
|
dict(name='tagNames', value=tag1.name),
|
|
dict(name='tagNames', value=tag2.name),
|
|
dict(name='chunk', value=part1, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.OK, response.content.decode()
|
|
|
|
output = test_directory / 'uploads/video.mp4'
|
|
assert output.is_file()
|
|
assert output.stat().st_size == 1_000_000
|
|
|
|
forms = [
|
|
dict(name='chunkNumber', value='1'),
|
|
dict(name='filename', value='video.mp4'),
|
|
dict(name='totalChunks', value='1'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value=56318),
|
|
dict(name='tagNames', value=tag1.name),
|
|
dict(name='tagNames', value=tag2.name),
|
|
dict(name='chunk', value=part2, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
|
|
|
|
assert output.is_file()
|
|
assert get_mimetype(output) == 'video/mp4'
|
|
assert output.stat().st_size == len(video_bytes)
|
|
assert output.read_bytes() == video_bytes
|
|
assert hashlib.md5(output.read_bytes()).hexdigest() == '2738c53bd7c01b01d408da11a55bfa36'
|
|
|
|
file_group: FileGroup = test_session.query(FileGroup).one()
|
|
assert file_group.mimetype == 'video/mp4'
|
|
assert set(file_group.tag_names) == {tag1.name, tag2.name}, 'Two tags should be applied.'
|
|
assert file_group.indexed, 'File should be indexed after upload.'
|
|
assert file_group.model, 'File should be modeled'
|
|
# Video was modeled, but has no Channel.
|
|
video: Video = test_session.query(Video).one()
|
|
assert video.video_path
|
|
assert not video.info_json_path
|
|
assert not video.channel and not video.channel_id
|
|
|
|
# Can't upload again because the file already exists.
|
|
forms = [
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='filename', value='video.mp4'),
|
|
dict(name='totalChunks', value='1'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value=1_000_000),
|
|
dict(name='chunk', value=part1, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST, response.content.decode()
|
|
assert response.json['error'] == 'File already exists!'
|
|
|
|
# FileGroup was not deleted.
|
|
file_group: FileGroup = test_session.query(FileGroup).one()
|
|
assert file_group
|
|
|
|
# Remove Tag so file can be uploaded again.
|
|
file_group.set_tags(test_session, [])
|
|
|
|
# An existing file can be overwritten.
|
|
forms = [
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='filename', value='video.mp4'),
|
|
dict(name='totalChunks', value='1'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value=1_000_000),
|
|
dict(name='chunk', value=part1, filename='chunk'),
|
|
dict(name='overwrite', value=True),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.OK, response.content.decode()
|
|
|
|
file_group: FileGroup = test_session.query(FileGroup).one_or_none()
|
|
assert not file_group, 'Old FileGroup should be deleted when new upload starts.'
|
|
|
|
# Upload the second part of the file again.
|
|
forms = [
|
|
dict(name='chunkNumber', value='1'),
|
|
dict(name='filename', value='video.mp4'),
|
|
dict(name='totalChunks', value='1'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value=56318),
|
|
dict(name='chunk', value=part2, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
|
|
|
|
# Make the same checks as those done above because the file was uploaded again.
|
|
assert output.is_file()
|
|
assert get_mimetype(output) == 'video/mp4'
|
|
assert output.stat().st_size == len(video_bytes)
|
|
assert output.read_bytes() == video_bytes
|
|
assert hashlib.md5(output.read_bytes()).hexdigest() == '2738c53bd7c01b01d408da11a55bfa36'
|
|
|
|
file_group: FileGroup = test_session.query(FileGroup).one()
|
|
assert file_group.mimetype == 'video/mp4'
|
|
assert not file_group.tag_names, 'No tags this time.'
|
|
assert file_group.indexed, 'File should be indexed after upload.'
|
|
assert file_group.model, 'File should be modeled'
|
|
assert isinstance(file_group.get_model_record(test_session), Video)
|
|
assert file_group.location == f'/videos/{file_group.id}'
|
|
# Video was modeled, but has no Channel.
|
|
video: Video = test_session.query(Video).one()
|
|
assert video.video_path
|
|
assert video.file_group == file_group
|
|
assert not video.info_json_path
|
|
assert not video.channel and not video.channel_id
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_upload_video_extracts_ffprobe(test_session, async_client, test_directory, make_files_structure,
|
|
make_multipart_form, video_bytes):
|
|
"""Uploading a video file should extract and store ffprobe data."""
|
|
make_files_structure(['uploads/'])
|
|
|
|
# Upload video in chunks (same pattern as test_post_upload)
|
|
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
|
|
|
|
# First chunk
|
|
forms = [
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='filename', value='test_ffprobe.mp4'),
|
|
dict(name='totalChunks', value='1'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value=1_000_000),
|
|
dict(name='chunk', value=part1, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
# Second chunk (final)
|
|
forms = [
|
|
dict(name='chunkNumber', value='1'),
|
|
dict(name='filename', value='test_ffprobe.mp4'),
|
|
dict(name='totalChunks', value='1'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value=len(part2)),
|
|
dict(name='chunk', value=part2, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
# Verify video was created
|
|
video: Video = test_session.query(Video).one()
|
|
assert video.video_path
|
|
|
|
# THIS IS THE BUG - ffprobe_json should be populated but it's not
|
|
assert video.ffprobe_json is not None, 'ffprobe_json should be extracted on upload'
|
|
assert 'streams' in video.ffprobe_json, 'ffprobe_json should contain streams'
|
|
assert 'format' in video.ffprobe_json, 'ffprobe_json should contain format'
|
|
assert video.file_group.length, 'Duration should be extracted from ffprobe'
|
|
assert video.ffprobe_json_path
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_upload_text(test_session, async_client, test_directory, make_files_structure, make_multipart_form):
|
|
"""A file that cannot be modeled can still be uploaded, and is indexed."""
|
|
make_files_structure(['uploads/'])
|
|
|
|
contents = 'hello'
|
|
forms = [
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='filename', value='the title.txt'),
|
|
dict(name='totalChunks', value='0'),
|
|
dict(name='destination', value='uploads'),
|
|
dict(name='chunkSize', value=5),
|
|
dict(name='chunk', value=contents, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
|
|
|
|
file_group: FileGroup = test_session.query(FileGroup).one()
|
|
assert file_group.indexed is True
|
|
assert file_group.size == 5
|
|
assert file_group.a_text == 'the title txt'
|
|
assert file_group.d_text == 'hello'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_upload_new_directory(test_session, async_client, test_directory, make_files_structure,
|
|
make_multipart_form,
|
|
tag_factory, video_bytes):
|
|
"""User can create a new directory when uploading."""
|
|
tag1, tag2 = await tag_factory(), await tag_factory()
|
|
|
|
forms = [
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='filename', value='video.mp4'),
|
|
dict(name='totalChunks', value='1'),
|
|
dict(name='destination', value='some new directory'),
|
|
dict(name='mkdir', value=True),
|
|
dict(name='chunkSize', value=len(video_bytes)),
|
|
dict(name='tagNames', value=tag1.name),
|
|
dict(name='tagNames', value=tag2.name),
|
|
dict(name='chunk', value=video_bytes, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.OK, response.content.decode()
|
|
|
|
assert (test_directory / 'some new directory').is_dir()
|
|
assert (test_directory / 'some new directory/video.mp4').is_file()
|
|
assert (test_directory / 'some new directory/video.mp4').read_bytes() == video_bytes
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_directory_crud(test_session, async_client, test_directory, assert_directories, assert_files,
|
|
refresh_files):
|
|
"""A directory can be created in a subdirectory. Errors are returned if there are conflicts."""
|
|
foo = test_directory / 'foo'
|
|
foo.mkdir()
|
|
await refresh_files()
|
|
assert_directories({'foo', })
|
|
|
|
# Create a subdirectory.
|
|
content = dict(path='foo/bar')
|
|
request, response = await async_client.post('/api/files/directory', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
assert (test_directory / 'foo/bar').is_dir()
|
|
assert_directories({'foo', 'foo/bar'})
|
|
|
|
# Cannot create twice.
|
|
content = dict(path='foo/bar')
|
|
request, response = await async_client.post('/api/files/directory', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.CONFLICT
|
|
assert (test_directory / 'foo/bar').is_dir()
|
|
assert_directories({'foo', 'foo/bar'})
|
|
|
|
# Create a file to be deleted.
|
|
(test_directory / 'foo/bar/asdf.txt').write_text('asdf')
|
|
|
|
# Can get information about a directory.
|
|
request, response = await async_client.post('/api/files/get_directory', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['path'] == 'foo/bar'
|
|
assert response.json['size'] == 4
|
|
assert response.json['file_count'] == 1
|
|
|
|
# Can rename the directory.
|
|
content = dict(path='foo/bar', new_name='baz')
|
|
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert not (test_directory / 'foo/bar').exists()
|
|
assert (test_directory / 'foo/baz').exists()
|
|
assert (test_directory / 'foo/baz/asdf.txt').exists()
|
|
|
|
# Deletion is recursive.
|
|
assert (test_directory / 'foo/baz/asdf.txt').is_file()
|
|
content = dict(paths=['foo/baz', ])
|
|
request, response = await async_client.post('/api/files/delete', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert not (test_directory / 'foo/baz').exists()
|
|
assert not (test_directory / 'foo/baz/asdf.txt').exists()
|
|
# Only top directory is left.
|
|
assert_directories({'foo', })
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_move(test_session, test_directory, make_files_structure, async_client, await_background_tasks):
|
|
"""Files can be moved up and down the media directory. Destination directories should already exist."""
|
|
baz, bar = make_files_structure({
|
|
'foo/bar.txt': 'bar',
|
|
'baz.txt': 'baz',
|
|
})
|
|
|
|
# qux directory does not exist
|
|
content = dict(paths=['foo', 'baz.txt'], destination='qux')
|
|
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
(test_directory / 'qux').mkdir()
|
|
(test_directory / 'quux/quuz').mkdir(parents=True)
|
|
|
|
# mv foo baz.txt qux
|
|
content = dict(paths=['foo', 'baz.txt'], destination='qux')
|
|
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
await await_background_tasks()
|
|
assert not bar.exists()
|
|
assert (test_directory / 'qux/foo/bar.txt').is_file()
|
|
|
|
# mv foo/bar.txt qux
|
|
content = dict(paths=['qux/foo/bar.txt', ], destination='qux')
|
|
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
await await_background_tasks()
|
|
assert not bar.exists()
|
|
assert (test_directory / 'qux/bar.txt').is_file()
|
|
assert (test_directory / 'qux/foo').is_dir()
|
|
|
|
# mv qux/bar.txt ./
|
|
content = dict(paths=['qux/bar.txt', ], destination='')
|
|
request, response = await async_client.post('/api/files/move', json=content)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
await await_background_tasks()
|
|
assert (test_directory / 'bar.txt').is_file()
|
|
|
|
# mv bar.txt quux/quuz/bar.txt
|
|
content = dict(paths=['bar.txt', ], destination='quux/quuz')
|
|
request, response = await async_client.post('/api/files/move', json=content)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
await await_background_tasks()
|
|
assert (test_directory / 'quux/quuz/bar.txt').is_file()
|
|
|
|
# mv quux/quuz/bar.txt quux/bar.txt
|
|
content = dict(paths=['quux/quuz/bar.txt', ], destination='quux')
|
|
request, response = await async_client.post('/api/files/move', json=content)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
await await_background_tasks()
|
|
assert (test_directory / 'quux/bar.txt').is_file()
|
|
assert (test_directory / 'qux/foo').is_dir()
|
|
assert (test_directory / 'quux/quuz').is_dir()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rename_file(test_session, test_directory, make_files_structure, async_client, refresh_files):
|
|
"""A FileGroup can be renamed. The title and search index is updated."""
|
|
foo_file, = make_files_structure({
|
|
'foo/bar/baz.txt': 'asdf',
|
|
})
|
|
await refresh_files()
|
|
foo_fg = test_session.query(FileGroup).one()
|
|
assert foo_fg.a_text == 'baz txt'
|
|
assert foo_fg.d_text == 'asdf'
|
|
|
|
# mv foo/bar/baz.txt foo/bar/qux.txt
|
|
content = dict(path='foo/bar/baz.txt', new_name='qux.txt')
|
|
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert not foo_file.exists()
|
|
assert (test_directory / 'foo/bar/qux.txt').is_file()
|
|
assert (test_directory / 'foo/bar/qux.txt').read_text() == 'asdf'
|
|
assert foo_fg.a_text == 'qux txt'
|
|
assert foo_fg.d_text == 'asdf'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rename_file_with_associated_files_via_api(test_session, test_directory, make_files_structure,
|
|
async_client, video_bytes, srt_text, refresh_files):
|
|
"""Renaming a FileGroup via API should rename all associated files.
|
|
|
|
Regression test: When renaming "example.mp4" to "example 2.mp4" and back via API,
|
|
associated files like "example.srt" should also be renamed both times.
|
|
"""
|
|
video, srt = make_files_structure({
|
|
'example.mp4': video_bytes,
|
|
'example.srt': srt_text,
|
|
})
|
|
await refresh_files()
|
|
|
|
# Verify initial state
|
|
fg = test_session.query(FileGroup).one()
|
|
assert fg.primary_path == video
|
|
# Video validation creates .ffprobe.json, so we have 3 files
|
|
assert len(fg.files) == 3
|
|
|
|
# First rename via API: "example.mp4" -> "example 2.mp4"
|
|
content = dict(path='example.mp4', new_name='example 2.mp4')
|
|
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Verify first rename - both files should be renamed
|
|
new_video_1 = test_directory / 'example 2.mp4'
|
|
new_srt_1 = test_directory / 'example 2.srt'
|
|
assert new_video_1.is_file(), "Video should be renamed to 'example 2.mp4'"
|
|
assert new_srt_1.is_file(), "SRT should be renamed to 'example 2.srt'"
|
|
assert not video.exists(), "Old video should not exist"
|
|
assert not srt.exists(), "Old SRT should not exist"
|
|
|
|
# Verify FileGroup is updated
|
|
fg = test_session.query(FileGroup).one()
|
|
assert fg.primary_path == new_video_1
|
|
file_names = [f['path'] for f in fg.files]
|
|
assert 'example 2.mp4' in file_names
|
|
assert 'example 2.srt' in file_names
|
|
assert len(fg.files) == 3
|
|
|
|
# Second rename via API: "example 2.mp4" -> "example.mp4"
|
|
content = dict(path='example 2.mp4', new_name='example.mp4')
|
|
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Verify second rename - BOTH files should be renamed back
|
|
final_video = test_directory / 'example.mp4'
|
|
final_srt = test_directory / 'example.srt'
|
|
assert final_video.is_file(), "Video should be renamed back to 'example.mp4'"
|
|
assert final_srt.is_file(), "SRT should be renamed back to 'example.srt'"
|
|
assert not new_video_1.exists(), "'example 2.mp4' should not exist"
|
|
assert not new_srt_1.exists(), "'example 2.srt' should not exist"
|
|
|
|
# Verify FileGroup is updated
|
|
fg = test_session.query(FileGroup).one()
|
|
assert fg.primary_path == final_video
|
|
file_names = [f['path'] for f in fg.files]
|
|
assert 'example.mp4' in file_names
|
|
assert 'example.srt' in file_names
|
|
assert len(fg.files) == 3
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rename_directory(test_session, test_directory, make_files_structure, async_client, refresh_files):
|
|
make_files_structure({
|
|
'foo/bar/baz.txt': 'asdf',
|
|
})
|
|
await refresh_files()
|
|
bar, foo = test_session.query(Directory).order_by(Directory.name).all()
|
|
assert foo.name == 'foo' and bar.name == 'bar'
|
|
|
|
# mv foo/bar foo/qux
|
|
content = dict(path='foo/bar', new_name='qux')
|
|
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert not (test_directory / 'foo/bar').exists()
|
|
assert (test_directory / 'foo/qux/baz.txt').is_file()
|
|
assert (test_directory / 'foo/qux/baz.txt').read_text() == 'asdf'
|
|
foo, qux = test_session.query(Directory).order_by(Directory.name).all()
|
|
assert foo.name == 'foo' and qux.name == 'qux'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rename_directory_stale_destination_fails_promptly(
|
|
test_session, test_directory, make_files_structure, async_client, refresh_files
|
|
):
|
|
"""A directory rename that collides with a stale FileGroup must fail promptly.
|
|
|
|
Regression: the move's IntegrityError left the job 'pending', so
|
|
POST /api/files/rename hung until wait_for_job's 300s timeout.
|
|
"""
|
|
import asyncio
|
|
import shutil
|
|
|
|
make_files_structure({'dir-a/one.txt': 'hello'})
|
|
await refresh_files()
|
|
|
|
content = dict(path='dir-a', new_name='dir-b')
|
|
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Remove dest on disk without refreshing; recreate the old source.
|
|
shutil.rmtree(test_directory / 'dir-b')
|
|
(test_directory / 'dir-a').mkdir()
|
|
(test_directory / 'dir-a' / 'one.txt').write_text('hello')
|
|
|
|
request, response = await asyncio.wait_for(
|
|
async_client.post('/api/files/rename', content=json.dumps(content)),
|
|
timeout=5,
|
|
)
|
|
assert response.status_code == HTTPStatus.CONFLICT, response.json
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rename_collection_directory(test_session, test_directory, make_files_structure, async_client,
|
|
refresh_files):
|
|
"""Renaming a Collection's directory updates the Collection.directory."""
|
|
from wrolpi.collections.models import Collection
|
|
|
|
# Create a collection with a directory
|
|
collection_dir = test_directory / 'my_collection'
|
|
collection_dir.mkdir()
|
|
(collection_dir / 'file.txt').write_text('contents')
|
|
collection = Collection(name='Test Collection', kind='channel', directory=collection_dir)
|
|
test_session.add(collection)
|
|
test_session.commit()
|
|
collection_id = collection.id
|
|
|
|
await refresh_files()
|
|
|
|
# Rename the directory
|
|
content = dict(path='my_collection', new_name='renamed_collection')
|
|
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Verify Collection.directory was updated
|
|
collection = test_session.query(Collection).filter_by(id=collection_id).one()
|
|
assert collection.directory == test_directory / 'renamed_collection'
|
|
assert not (test_directory / 'my_collection').exists()
|
|
assert (test_directory / 'renamed_collection' / 'file.txt').is_file()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rename_directory_with_nested_collections(test_session, test_directory, async_client, refresh_files):
|
|
"""Renaming a parent directory updates nested Collection directories and download destinations."""
|
|
from wrolpi.collections.models import Collection
|
|
from wrolpi.downloader import Download
|
|
|
|
# Create parent directory with nested collection directories
|
|
parent_dir = test_directory / 'videos'
|
|
parent_dir.mkdir()
|
|
|
|
channel_a_dir = parent_dir / 'channel_a'
|
|
channel_a_dir.mkdir()
|
|
(channel_a_dir / 'video1.mp4').write_bytes(b'fake video 1')
|
|
|
|
channel_b_dir = parent_dir / 'channel_b'
|
|
channel_b_dir.mkdir()
|
|
(channel_b_dir / 'video2.mp4').write_bytes(b'fake video 2')
|
|
|
|
# Create two channel collections for the nested directories
|
|
coll_a = Collection(name='Channel A', kind='channel', directory=channel_a_dir)
|
|
coll_b = Collection(name='Channel B', kind='channel', directory=channel_b_dir)
|
|
test_session.add_all([coll_a, coll_b])
|
|
test_session.commit()
|
|
|
|
# Create a download associated with channel_a
|
|
download = Download(url='https://example.com/channel_a', destination=channel_a_dir, collection_id=coll_a.id)
|
|
test_session.add(download)
|
|
test_session.commit()
|
|
coll_a_id, coll_b_id, download_id = coll_a.id, coll_b.id, download.id
|
|
|
|
await refresh_files()
|
|
|
|
# Rename the parent directory
|
|
content = dict(path='videos', new_name='my_videos')
|
|
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Verify nested Collection directories were updated
|
|
coll_a = test_session.query(Collection).filter_by(id=coll_a_id).one()
|
|
coll_b = test_session.query(Collection).filter_by(id=coll_b_id).one()
|
|
|
|
assert coll_a.directory == test_directory / 'my_videos' / 'channel_a'
|
|
assert coll_b.directory == test_directory / 'my_videos' / 'channel_b'
|
|
|
|
# Verify download destination was updated
|
|
download = test_session.query(Download).filter_by(id=download_id).one()
|
|
assert download.destination == test_directory / 'my_videos' / 'channel_a'
|
|
|
|
# Verify files were moved
|
|
assert (test_directory / 'my_videos' / 'channel_a' / 'video1.mp4').exists()
|
|
assert (test_directory / 'my_videos' / 'channel_b' / 'video2.mp4').exists()
|
|
assert not (test_directory / 'videos').exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_directory_recursive(test_session, test_directory, make_files_structure, async_client,
|
|
assert_files, refresh_files):
|
|
make_files_structure(['dir/foo', 'dir/bar', 'empty'])
|
|
await refresh_files()
|
|
assert test_session.query(FileGroup).count() == 3
|
|
|
|
content = {'paths': ['dir/', ]}
|
|
request, response = await async_client.post('/api/files/delete', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert test_session.query(FileGroup).count() == 1
|
|
|
|
empty = test_session.query(FileGroup).one()
|
|
assert empty.primary_path.name == 'empty'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_file(test_session, async_client, test_directory, make_files_structure, refresh_files):
|
|
"""Can get info about a single file."""
|
|
make_files_structure({'foo/bar.txt': 'foo contents'})
|
|
await refresh_files()
|
|
|
|
content = dict(file='foo/bar.txt')
|
|
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file']
|
|
assert response.json['file']['path'] == 'foo/bar.txt'
|
|
|
|
fg, = test_session.query(FileGroup).all()
|
|
assert fg.primary_path == test_directory / 'foo/bar.txt'
|
|
assert isinstance(fg.viewed, datetime)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ignore_directory(test_session, async_client, test_directory, make_files_structure, test_wrolpi_config,
|
|
await_switches, refresh_files):
|
|
"""A maintainer can ignore/un-ignore directories. The files in the directory should not be refreshed."""
|
|
# Remove any default ignored directories.
|
|
get_wrolpi_config().ignored_directories = []
|
|
|
|
foo, bar, baz = make_files_structure(['foo/foo.txt', 'foo/bar.txt', 'baz/baz.txt'])
|
|
assert len(get_wrolpi_config().ignored_directories) == 0
|
|
|
|
# Ignore baz/
|
|
content = dict(path=str(baz.parent))
|
|
request, response = await async_client.post('/api/files/ignore_directory', json=content)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(get_wrolpi_config().ignored_directories) == 1
|
|
|
|
await refresh_files()
|
|
|
|
files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
|
assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'wrolpi.yaml'}
|
|
|
|
# Un-ignore baz/
|
|
request, response = await async_client.post('/api/files/unignore_directory', json=content)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(get_wrolpi_config().ignored_directories) == 0
|
|
|
|
await refresh_files()
|
|
|
|
files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
|
# Ignore configs.
|
|
files = {i for i in files if not i.primary_path.name.endswith('.yaml')}
|
|
assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'baz.txt'}
|
|
|
|
# Cannot ignore special directories.
|
|
content = dict(path=str(test_directory / 'videos'))
|
|
request, response = await async_client.post('/api/files/ignore_directory', json=content)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert len(get_wrolpi_config().ignored_directories) == 0
|
|
|
|
# Zims directory depends on FileGroup indexing for Manage/search — cannot ignore.
|
|
content = dict(path=str(test_directory / 'zims'))
|
|
request, response = await async_client.post('/api/files/ignore_directory', json=content)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert len(get_wrolpi_config().ignored_directories) == 0
|
|
|
|
# Relative form of zims is also refused.
|
|
content = dict(path='zims')
|
|
request, response = await async_client.post('/api/files/ignore_directory', json=content)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert len(get_wrolpi_config().ignored_directories) == 0
|
|
|
|
# Map is allowed to be ignored (lists PMTiles from the filesystem, not FileGroups).
|
|
content = dict(path=str(test_directory / 'map'))
|
|
request, response = await async_client.post('/api/files/ignore_directory', json=content)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert any(pathlib.Path(i).name == 'map' or str(i).endswith('/map')
|
|
for i in get_wrolpi_config().ignored_directories)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sanitize_ignored_zims_directory(test_session, async_client, test_directory, test_wrolpi_config):
|
|
"""Zims previously saved as ignored is stripped so refresh can index them again."""
|
|
from wrolpi.files.lib import (
|
|
get_normalized_ignored_directories,
|
|
sanitize_ignored_directories,
|
|
)
|
|
|
|
# Simulate an older config that incorrectly ignored zims (and intentionally ignored map).
|
|
get_wrolpi_config().ignored_directories = ['config', 'tags', 'map', 'zims']
|
|
|
|
# Normalized list used by refresh must never exclude zims.
|
|
normalized = get_normalized_ignored_directories()
|
|
assert not any(pathlib.Path(i).name == 'zims' for i in normalized)
|
|
assert any(pathlib.Path(i).name == 'map' for i in normalized)
|
|
|
|
sanitize_ignored_directories()
|
|
assert 'zims' not in get_wrolpi_config().ignored_directories
|
|
assert 'map' in get_wrolpi_config().ignored_directories
|
|
|
|
# Settings endpoint also sanitizes so the UI eye-icon matches.
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
ignored = [str(i) for i in response.json['ignored_directories']]
|
|
assert 'zims' not in ignored
|
|
assert 'map' in ignored
|
|
|
|
|
|
# --- Bulk Tagging API Tests ---
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bulk_tag_preview_api(test_session, async_client, make_files_structure, tag_factory):
|
|
"""The bulk tag preview endpoint returns file count and shared tags."""
|
|
foo, bar, baz = make_files_structure({
|
|
'foo.txt': 'foo',
|
|
'bar.txt': 'bar',
|
|
'baz.txt': 'baz',
|
|
})
|
|
|
|
# Create FileGroups
|
|
fg_foo = FileGroup.from_paths(test_session, foo)
|
|
fg_bar = FileGroup.from_paths(test_session, bar)
|
|
fg_baz = FileGroup.from_paths(test_session, baz)
|
|
test_session.commit()
|
|
|
|
# Add shared and non-shared tags
|
|
tag1 = await tag_factory('shared_tag')
|
|
tag2 = await tag_factory('only_foo')
|
|
fg_foo.add_tag(test_session, tag1.id)
|
|
fg_foo.add_tag(test_session, tag2.id)
|
|
fg_bar.add_tag(test_session, tag1.id)
|
|
fg_baz.add_tag(test_session, tag1.id)
|
|
test_session.commit()
|
|
|
|
content = {'paths': ['foo.txt', 'bar.txt', 'baz.txt']}
|
|
request, response = await async_client.post('/api/files/bulk_tag/preview', json=content)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_count'] == 3
|
|
assert 'shared_tag' in response.json['shared_tag_names']
|
|
assert 'only_foo' not in response.json['shared_tag_names']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bulk_tag_preview_empty(test_session, async_client):
|
|
"""The bulk tag preview endpoint handles empty paths."""
|
|
content = {'paths': []}
|
|
request, response = await async_client.post('/api/files/bulk_tag/preview', json=content)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_count'] == 0
|
|
assert response.json['shared_tag_names'] == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bulk_tag_apply_api(test_session, async_client, make_files_structure, tag_factory):
|
|
"""The bulk tag apply endpoint queues a tagging job."""
|
|
foo, bar = make_files_structure({
|
|
'foo.txt': 'foo',
|
|
'bar.txt': 'bar',
|
|
})
|
|
|
|
# Create FileGroups
|
|
FileGroup.from_paths(test_session, foo)
|
|
FileGroup.from_paths(test_session, bar)
|
|
test_session.commit()
|
|
|
|
# Create a tag
|
|
tag = await tag_factory('api_tag')
|
|
test_session.commit()
|
|
|
|
content = {
|
|
'paths': ['foo.txt', 'bar.txt'],
|
|
'add_tag_names': ['api_tag'],
|
|
'remove_tag_names': [],
|
|
}
|
|
request, response = await async_client.post('/api/files/bulk_tag/apply', json=content)
|
|
assert response.status_code == HTTPStatus.ACCEPTED
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_bulk_tag_progress_api(test_session, async_client):
|
|
"""The bulk tag progress endpoint returns progress information."""
|
|
request, response = await async_client.get('/api/files/bulk_tag/progress')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert 'status' in response.json
|
|
assert 'total' in response.json
|
|
assert 'completed' in response.json
|
|
assert 'queued_jobs' in response.json
|
|
|
|
|
|
# --- File Tracking Tests ---
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_file_skip_tracking(test_session, async_client, test_directory, make_files_structure, refresh_files):
|
|
"""Can get info about a single file without tracking the view."""
|
|
make_files_structure({'foo/bar.txt': 'foo contents'})
|
|
await refresh_files()
|
|
|
|
# Request the file with skip_tracking=True
|
|
content = dict(file='foo/bar.txt', skip_tracking=True)
|
|
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file']
|
|
assert response.json['file']['path'] == 'foo/bar.txt'
|
|
|
|
# FileGroup.viewed should not be set
|
|
fg, = test_session.query(FileGroup).all()
|
|
assert fg.primary_path == test_directory / 'foo/bar.txt'
|
|
assert fg.viewed is None, "FileGroup.viewed should not be set when skip_tracking=True"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_file_ignored_directory_not_tracked(test_session, async_client, test_directory, make_files_structure,
|
|
refresh_files, test_wrolpi_config, await_switches):
|
|
"""Files in ignored directories do not get their viewed timestamp set."""
|
|
# Create files in a directory that will be ignored
|
|
foo, bar = make_files_structure({'config/settings.txt': 'settings', 'normal/file.txt': 'contents'})
|
|
|
|
# Ignore the config directory
|
|
get_wrolpi_config().ignored_directories = [str(test_directory / 'config')]
|
|
await await_switches()
|
|
|
|
# Refresh only the normal directory (config is ignored)
|
|
await refresh_files()
|
|
|
|
# The normal file should be in the database
|
|
normal_fg = test_session.query(FileGroup).filter(FileGroup.primary_path.like('%normal%')).one()
|
|
assert normal_fg.viewed is None
|
|
|
|
# Request the normal file - it should be tracked
|
|
content = dict(file='normal/file.txt')
|
|
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
test_session.refresh(normal_fg)
|
|
assert normal_fg.viewed is not None, "Normal files should be tracked"
|
|
|
|
# Request a file in the ignored directory - it should return the file data
|
|
# but should not be tracked (since it's in an ignored directory)
|
|
content = dict(file='config/settings.txt')
|
|
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
# Verify the ignored file is not in the database (because it was never refreshed)
|
|
ignored_fgs = test_session.query(FileGroup).filter(FileGroup.primary_path.like('%config%')).all()
|
|
assert len(ignored_fgs) == 0, "Files in ignored directories should not be in the database"
|