wrolpi/wrolpi/files/test/test_api.py
Roland 85f22b61ca
files: a failed directory rename that hung for five minutes
A directory rename waits inline on a move job. When that move failed
(stale FileGroup rows colliding on primary_path), handle_move reverted
the files and sent a failure event but left the job pending. wait_for_job
only understands 'complete', so POST /api/files/rename spun until its
300s timeout and the Rename modal never closed.

The same sequence could kill file processing for the life of the process.
perpetual_signal treated any CancelledError as terminal and logged
nothing. Separately, perpetual_tasks_started is a shared Event that
survives the worker that set it: docker auto-reloads and worker crashes
left every later after_server_start as a no-op. Queued refreshes sat
untouched while worker_status reported idle. Restarting the API was the
only recovery.

Mark failed jobs failed and raise immediately from wait_for_job. Catch
unexpected cancellation and reschedule unless the app is actually
stopping. Track the perpetual-loop owner PID and retake the claim when
that process is gone; every Sanic worker still runs the file-worker
pump. Harden transfer_queue against the cross-process Empty race, and
stop request processes stealing from the shared public queue.
2026-08-19 18:57:17 -06:00

1635 lines
72 KiB
Python

import hashlib
import json
import pathlib
from datetime import datetime, timezone
from http import HTTPStatus
from unittest import mock
import pytest
from modules.videos import Video
from wrolpi.common import get_wrolpi_config
from wrolpi.files.lib import get_mimetype
from wrolpi.files.models import FileGroup, Directory
from wrolpi.tags import TagFile
from wrolpi.test.common import assert_dict_contains
from wrolpi.vars import PROJECT_DIR
@pytest.mark.asyncio
async def test_list_files_api(test_session, async_client, make_files_structure, test_directory):
files = [
'archives/bar.txt',
'archives/baz/bar.txt',
'archives/baz/foo.txt',
'archives/foo.txt',
'empty directory/',
'videos/other video.mp4',
'videos/some video.mp4',
'lost+found/', # Should always be ignored.
]
files = make_files_structure(files)
files[0].write_text('bar contents')
async def check_get_files(directories, expected_files):
request, response = await async_client.post('/api/files', content=json.dumps({'directories': directories}))
assert response.status_code == HTTPStatus.OK
assert not response.json.get('errors')
# The first dict is the media directory.
children = response.json['files']
assert_dict_contains(children, expected_files)
# Requesting no directories results in the top-level results.
expected = {
'archives/': {'path': 'archives/', 'is_empty': False},
'empty directory/': {'path': 'empty directory/', 'is_empty': True},
'videos/': {'path': 'videos/', 'is_empty': False}
}
await check_get_files([], expected)
# empty directory is empty
expected = {
'archives/': {'path': 'archives/', 'is_empty': False},
'empty directory/': {'path': 'empty directory/', 'children': {}, 'is_empty': True},
'videos/': {'path': 'videos/', 'is_empty': False}
}
await check_get_files(['empty directory'], expected)
expected = {
'archives/': {
'path': 'archives/',
'children': {
'foo.txt': {'path': 'archives/foo.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
'bar.txt': {'path': 'archives/bar.txt', 'size': 12, 'mimetype': 'text/plain'},
'baz/': {'path': 'archives/baz/', 'is_empty': False},
},
'is_empty': False,
},
'empty directory/': {'path': 'empty directory/', 'is_empty': True},
'videos/': {'path': 'videos/', 'is_empty': False}
}
await check_get_files(['archives'], expected)
# Sub-directories are supported.
expected = {
'archives/': {
'path': 'archives/',
'children': {
'foo.txt': {'path': 'archives/foo.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
'bar.txt': {'path': 'archives/bar.txt', 'size': 12, 'mimetype': 'text/plain'},
'baz/': {
'path': 'archives/baz/',
'children': {
'bar.txt': {'path': 'archives/baz/bar.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
'foo.txt': {'path': 'archives/baz/foo.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
},
'is_empty': False,
}
},
'is_empty': False
},
'empty directory/': {'path': 'empty directory/', 'is_empty': True},
'videos/': {'path': 'videos/', 'is_empty': False},
}
await check_get_files(['archives', 'archives/baz'], expected)
# Requesting only a subdirectory also returns `archives` contents.
await check_get_files(['archives/baz'], expected)
expected = {
'archives/': {
'path': 'archives/',
'children': {
'foo.txt': {'path': 'archives/foo.txt', 'size': 0, 'mimetype': 'inode/x-empty'},
'bar.txt': {'path': 'archives/bar.txt', 'size': 12, 'mimetype': 'text/plain'},
'baz/': {'path': 'archives/baz/', 'is_empty': False},
}, 'is_empty': False
},
'empty directory/': {'path': 'empty directory/', 'is_empty': True},
'videos/': {
'path': 'videos/',
'children': {
'other video.mp4': {'path': 'videos/other video.mp4', 'size': 0, 'mimetype': 'inode/x-empty'},
'some video.mp4': {'path': 'videos/some video.mp4', 'size': 0, 'mimetype': 'inode/x-empty'},
},
'is_empty': False
}
}
await check_get_files(['archives', 'videos'], expected)
# Order does not matter.
await check_get_files(['videos', 'archives'], expected)
@pytest.mark.asyncio
async def test_list_files_api_hides_ds_store(test_session, async_client, make_files_structure, test_directory):
"""macOS metadata files (.DS_Store / ._.DS_Store) are never shown in the FileBrowser."""
make_files_structure([
'.DS_Store', # Top-level metadata file should be hidden.
'._.DS_Store', # Top-level AppleDouble metadata file should be hidden.
'archives/foo.txt',
'archives/.DS_Store', # Nested metadata file should be hidden.
'archives/._.DS_Store', # Nested AppleDouble metadata file should be hidden.
])
async def get_files(directories):
request, response = await async_client.post('/api/files', content=json.dumps({'directories': directories}))
assert response.status_code == HTTPStatus.OK
return response.json['files']
# Top-level listing hides the metadata files.
top_level = await get_files([])
assert '.DS_Store' not in top_level
assert '._.DS_Store' not in top_level
assert 'archives/' in top_level
# Nested listing hides the metadata files, but still shows real files.
nested = await get_files(['archives'])
children = nested['archives/']['children']
assert '.DS_Store' not in children
assert '._.DS_Store' not in children
assert 'foo.txt' in children
@pytest.mark.asyncio
async def test_delete_file(test_session, async_client, make_files_structure, test_directory):
files = ['bar.txt', 'baz/', 'foo']
make_files_structure(files)
# Delete a file.
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['bar.txt']}))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not (test_directory / 'bar.txt').is_file()
assert (test_directory / 'baz').is_dir()
# Delete a directory.
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['baz']}))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not (test_directory / 'bar.txt').is_file()
assert not (test_directory / 'baz').is_dir()
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['bad file']}))
assert response.status_code == HTTPStatus.NO_CONTENT
@pytest.mark.asyncio
@pytest.mark.parametrize(
'paths', [
[],
['', ],
]
)
async def test_delete_invalid_file(async_client, paths):
"""Some paths must be passed."""
with mock.patch('wrolpi.files.api.lib.delete') as mock_delete_file:
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': paths}))
assert response.status_code == HTTPStatus.BAD_REQUEST
mock_delete_file.assert_not_called()
@pytest.mark.asyncio
async def test_delete_wrol_mode(async_client, wrol_mode_fixture):
"""Can't delete a file when WROL Mode is enabled."""
await wrol_mode_fixture(True)
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['foo', ]}))
assert response.status_code == HTTPStatus.FORBIDDEN
assert response.json['code'] == 'WROL_MODE_ENABLED'
@pytest.mark.asyncio
async def test_delete_file_groups_mixed_batch(async_client, test_session, video_factory, archive_factory):
"""A single POST /api/files/delete_groups can delete a mixture of model types in one call."""
from modules.archive.models import Archive
video = video_factory(with_video_file=True)
archive = archive_factory('example.com', 'https://example.com/1')
test_session.commit()
video_paths = list(video.file_group.my_paths())
archive_paths = list(archive.my_paths())
body = json.dumps({'file_group_ids': [video.file_group_id, archive.file_group_id]})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.NO_CONTENT
assert all(not p.exists() for p in video_paths)
assert all(not p.exists() for p in archive_paths)
assert test_session.query(Video).count() == 0
assert test_session.query(Archive).count() == 0
assert test_session.query(FileGroup).count() == 0
@pytest.mark.asyncio
async def test_delete_file_groups_default_deleter(async_client, test_session, make_files_structure,
refresh_files):
"""A FileGroup with no specialized model (e.g., a plain text file) is removed via FileGroup.delete."""
foo, = make_files_structure({'foo.txt': 'hello'})
await refresh_files()
fg = test_session.query(FileGroup).one()
assert fg.model is None # plain text isn't modeled by Video/Archive/Doc/Zim
fg_id = fg.id
body = json.dumps({'file_group_ids': [fg_id]})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.NO_CONTENT
assert not foo.exists()
assert test_session.query(FileGroup).filter_by(id=fg_id).one_or_none() is None
@pytest.mark.asyncio
async def test_delete_file_groups_unknown_ids(async_client, test_session):
"""Deleting non-existent IDs returns 404 from UnknownFile."""
body = json.dumps({'file_group_ids': [99999]})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.NOT_FOUND
assert response.json['code'] == 'UNKNOWN_FILE'
@pytest.mark.asyncio
async def test_delete_file_groups_empty_ids(async_client):
"""An empty file_group_ids list returns 400 (InvalidFile)."""
body = json.dumps({'file_group_ids': []})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_files_search_recent(test_session, test_directory, async_client, video_file_factory, refresh_files):
"""File search can return the most recently viewed files."""
video_file_factory(test_directory / 'foo.mp4'), video_file_factory(test_directory / 'bar.mp4')
baz = (test_directory / 'baz.txt')
baz.write_text('baz contents')
await refresh_files()
# baz is most recently viewed, foo has not been viewed.
bar, baz, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
bar.set_viewed(datetime(2000, 1, 1, 1, 1, 1, tzinfo=timezone.utc))
baz.set_viewed(datetime(2000, 1, 1, 1, 1, 2, tzinfo=timezone.utc))
test_session.commit()
# Results match the viewed order. foo is not viewed.
body = dict(order='viewed')
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], response.json
assert [i['name'] for i in response.json['file_groups']] == ['baz.txt', 'bar.mp4']
# Results match the reverse viewed order. foo is not viewed.
body = dict(order='-viewed')
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], response.json
assert [i['name'] for i in response.json['file_groups']] == ['bar.mp4', 'baz.txt']
# foo is included because `search_str` has a value.
body = dict(search_str='foo', order='viewed')
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], response.json
assert [i['name'] for i in response.json['file_groups']] == ['foo.mp4']
@pytest.mark.asyncio
async def test_files_search_null_params(test_session, test_directory, async_client, video_file_factory,
refresh_files):
"""Explicit JSON nulls (as the dashboard sends) must not break the search.
Regression test: the frontend sends `{"search_str": null, "offset": null, ...}`; explicit
nulls bypass the schema defaults and reached `OFFSET NULL`, which Postgres treated as 0 but
SQLite rejects with 'datatype mismatch'."""
video_file_factory(test_directory / 'foo.mp4')
await refresh_files()
file_group = test_session.query(FileGroup).one()
file_group.set_viewed(datetime(2000, 1, 1, 1, 1, 1, tzinfo=timezone.utc))
test_session.commit()
# The exact dashboard "recent files" payload.
body = {'search_str': None, 'offset': None, 'limit': 12, 'any_tag': False, 'tag_names': [], 'order': 'viewed'}
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK, response.json
assert [i['name'] for i in response.json['file_groups']] == ['foo.mp4']
# Null limit as well.
body = {'search_str': None, 'offset': None, 'limit': None}
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK, response.json
assert response.json['file_groups']
@pytest.mark.asyncio
async def test_files_search_stemming(test_session, async_client, make_files_structure, refresh_files,
assert_files_search):
"""Searching any grammatical variant of a word finds files named with other variants.
End-to-end stemming contract: "watch" must find "watching", "fishes" must find "fishing"."""
make_files_structure(['watching birds.mp4', 'fishing guide.txt', 'two cats.txt', 'unrelated.txt'])
await refresh_files()
await assert_files_search('watch', [dict(name='watching birds.mp4')])
await assert_files_search('watched', [dict(name='watching birds.mp4')])
await assert_files_search('fish', [dict(name='fishing guide.txt')])
await assert_files_search('fishes', [dict(name='fishing guide.txt')])
await assert_files_search('cat', [dict(name='two cats.txt')])
# Stemmers do not lemmatize: derived words are their own terms (same as the old engine).
await assert_files_search('runner', [])
@pytest.mark.asyncio
async def test_files_search(test_session, async_client, make_files_structure, assert_files_search):
# You can search an empty directory.
await assert_files_search('nothing', [])
# Create files in the temporary directory. Add some contents so the mimetype can be tested.
files = [
'foo_is_the_name.txt',
'archives/bar.txt',
'baz.mp4',
'baz baz two.mp4'
]
foo, bar, baz, baz2 = make_files_structure(files)
foo.write_text('foo contents')
bar.write_text('the bar contents')
baz.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
baz2.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes())
# Refresh so files can be searched.
request, response = await async_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
await assert_files_search('foo', [dict(primary_path='foo_is_the_name.txt')])
await assert_files_search('bar', [dict(primary_path='archives/bar.txt')])
await assert_files_search('baz', [dict(primary_path='baz baz two.mp4'), dict(primary_path='baz.mp4')])
await assert_files_search('two', [dict(primary_path='baz baz two.mp4')])
await assert_files_search('nothing', [])
@pytest.mark.asyncio
async def test_files_search_deep(test_session, async_client, make_files_structure):
"""Search defaults to the fast ABC column (title/author/description); `deep` opts into d_text
(captions/file contents)."""
guide, lecture = make_files_structure([
'gardening guide.txt',
'lecture.txt',
])
guide.write_text('planting tips')
# "gardening" only appears in this file's contents (d_text), not its name.
lecture.write_text('a lecture about gardening')
request, response = await async_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
async def search(**kwargs):
request, response = await async_client.post('/api/files/search', json=kwargs)
assert response.status_code == HTTPStatus.OK
return response.json['file_groups']
# Default (fast) search only matches the title.
results = await search(search_str='gardening')
assert [i['primary_path'] for i in results] == ['gardening guide.txt']
# Deep search also matches file contents; the title match ranks higher (weight A vs D).
results = await search(search_str='gardening', deep=True)
assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
# Headlines work in both modes.
results = await search(search_str='gardening', headline=True)
assert [i['primary_path'] for i in results] == ['gardening guide.txt']
assert '<b>gardening</b>' in results[0]['title_headline']
results = await search(search_str='gardening', deep=True, headline=True)
assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
assert '<b>gardening</b>' in results[1]['d_headline']
# The estimate endpoint returns both counts: `file_groups` matches the default fast search, and
# `file_groups_deep` shows what deep search would find, so the UI can offer deep search.
request, response = await async_client.post('/api/search_file_estimates',
json=dict(search_str='gardening'))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == 1
assert response.json['file_groups_deep'] == 2
@pytest.mark.asyncio
async def test_files_search_by_suffix(test_session, async_client, make_files_structure):
"""Files can be filtered by their primary file's suffix (used by the firmware flasher to find .bin files)."""
files = [
'software/MALVEKE.ino.bin',
'software/bffb/esp32_marauder.ino.bootloader.bin',
'software/firmware-heltec.uf2',
'software/firmware-r1-neo.uf2',
'notes.txt',
]
created = make_files_structure(files)
for path in created:
path.write_bytes(b'\x00\x01\x02\x03')
request, response = await async_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
async def search(**kwargs):
request, response = await async_client.post('/api/files/search', json=kwargs)
assert response.status_code == HTTPStatus.OK
return sorted(fg['primary_path'] for fg in response.json['file_groups'])
# Exact suffix match, normalized whether or not a leading dot is given, and case-insensitive.
assert await search(suffix='.bin') == ['software/MALVEKE.ino.bin',
'software/bffb/esp32_marauder.ino.bootloader.bin']
assert await search(suffix='bin') == ['software/MALVEKE.ino.bin',
'software/bffb/esp32_marauder.ino.bootloader.bin']
assert await search(suffix='.BIN') == ['software/MALVEKE.ino.bin',
'software/bffb/esp32_marauder.ino.bootloader.bin']
assert await search(suffix='.uf2') == ['software/firmware-heltec.uf2', 'software/firmware-r1-neo.uf2']
assert await search(suffix='.txt') == ['notes.txt']
# A suffix that matches nothing returns no results.
assert await search(suffix='.gguf') == []
# The `path` filter matches anywhere in the primary_path (case-insensitive), so a directory name works.
assert await search(path='bffb') == ['software/bffb/esp32_marauder.ino.bootloader.bin']
assert await search(path='BFFB') == ['software/bffb/esp32_marauder.ino.bootloader.bin']
assert await search(path='software/') == ['software/MALVEKE.ino.bin',
'software/bffb/esp32_marauder.ino.bootloader.bin',
'software/firmware-heltec.uf2', 'software/firmware-r1-neo.uf2']
# `suffix` and `path` combine (the firmware picker sends both).
assert await search(suffix='.bin', path='bffb') == ['software/bffb/esp32_marauder.ino.bootloader.bin']
@pytest.mark.asyncio
async def test_files_search_attaches_doc_section_hint(async_client, test_session, test_directory,
example_epub, refresh_files):
"""The global /api/files/search endpoint attaches `section_hint` to doc-modeled
results so the UI can deep-link into the matching chapter. Regression test for a
bug where only the docs-specific endpoint (/api/docs/search) attached hints."""
await refresh_files()
# "chapter" only appears in the EPUB body (d_text), so deep search is required to match it.
body = dict(search_str='chapter', deep=True)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
epub_results = [r for r in response.json['file_groups']
if r.get('model') == 'doc'
and (r.get('mimetype') or '').startswith('application/epub')]
assert len(epub_results) == 1, f'Expected 1 EPUB result, got: {response.json}'
result = epub_results[0]
assert 'section_hint' in result, \
f'doc result has no section_hint on global search path: {result}'
hint = result['section_hint']
assert hint['kind'] == 'epub_spine'
# The test EPUB's spine order is [titlepage, start, chapter1]; the chapter1
# document holds the "Chapter one" body, so its spine ordinal is 2.
assert hint['ordinal'] == 2
assert hint['label']
# Snippet uses our sentinel markers, not raw HTML.
assert '[[WROLPI_HL]]' in hint['snippet']
@pytest.mark.asyncio
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory, refresh_files):
one, two = await tag_factory(), await tag_factory()
files = [
'foo.txt',
'foo bar.txt',
'bar.txt',
]
make_files_structure(files)
await refresh_files()
bar, foobar, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
assert bar.primary_path.name == 'bar.txt' \
and foo.primary_path.name == 'foo.txt' \
and foobar.primary_path.name == 'foo bar.txt'
foo.add_tag(test_session, one.id)
foobar.add_tag(test_session, two.id)
test_session.commit()
# Only `foo` is tagged with `one`
body = dict(search_str='foo', tag_names=['one'])
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt'}
# Both `foo bar` and `foo` are tagged.
body = dict(search_str='foo', any_tag=True)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt', 'foo bar.txt'}
# `bar` is not tagged.
body = dict(search_str='bar', any_tag=True)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo bar.txt'}
# Cannot search for any_tag, and tag names.
body = dict(search_str='foo', tag_names=['one'], any_tag=True)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_refresh_files_list(test_session, async_client, make_files_structure, test_directory, video_bytes):
"""The user can request to refresh specific files."""
make_files_structure({
'bar.txt': 'hello',
'bar.mp4': video_bytes,
})
# Only the single file that was refreshed is discovered.
content = json.dumps({'paths': ['bar.txt']})
request, response = await async_client.post('/api/files/refresh', content=content)
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(FileGroup).count() == 1
group: FileGroup = test_session.query(FileGroup).one()
assert len(group.files) == 1
request, response = await async_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
group: FileGroup = test_session.query(FileGroup).one()
# Video validation creates .ffprobe.json, so we may have 2 or 3 files
assert len(group.files) >= 2
@pytest.mark.asyncio
async def test_file_statistics(test_session, async_client, test_directory, example_pdf, example_mobi, example_epub,
video_file):
"""A summary of File statistics can be fetched."""
# Give each file a unique stem.
video_file.rename(test_directory / 'video.mp4')
example_pdf.rename(test_directory / 'pdf.pdf')
example_mobi.rename(test_directory / 'mobi.mobi')
example_epub.rename(test_directory / 'epub.epub')
# Statistics can be fetched while empty.
request, response = await async_client.get('/api/statistics')
assert response.status_code == HTTPStatus.OK
assert response.json['file_statistics'] == {
'archive_count': 0,
'audio_count': 0,
'ebook_count': 0,
'image_count': 0,
'pdf_count': 0,
'tagged_files': 0,
'tagged_zims': 0,
'tags_count': 0,
'total_count': 0,
'total_size': 0,
'video_count': 0,
'zip_count': 0,
}
await async_client.post('/api/files/refresh')
request, response = await async_client.get('/api/statistics')
assert response.status_code == HTTPStatus.OK
stats = response.json['file_statistics']
stats.pop('total_size')
assert stats == {
'archive_count': 0,
'audio_count': 0,
'ebook_count': 2,
'image_count': 0,
'pdf_count': 1,
'tagged_files': 0,
'tagged_zims': 0,
'tags_count': 0,
'total_count': 7, # extracted covers (epub + pdf) + .ffprobe.json
'video_count': 1,
'zip_count': 0,
}
@pytest.mark.asyncio
async def test_file_group_tag_by_primary_path(test_session, async_client, test_directory, example_singlefile,
tag_factory, insert_file_group):
singlefile = FileGroup.from_paths(test_session, example_singlefile)
tag1 = await tag_factory()
tag2 = await tag_factory()
test_session.commit()
# FileGroup can be tagged with its primary_path.
content = dict(file_group_primary_path=str(singlefile.primary_path.relative_to(test_directory)), tag_name=tag1.name)
request, response = await async_client.post('/api/files/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(TagFile).count() == 1
# FileGroup can be tagged with its id.
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
request, response = await async_client.post('/api/files/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(TagFile).count() == 2
# FileGroup can be untagged with its primary_path.
content = dict(file_group_id=singlefile.id, tag_id=tag1.id)
request, response = await async_client.post('/api/files/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(TagFile).count() == 1
# FileGroup can be untagged with its id.
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
request, response = await async_client.post('/api/files/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(TagFile).count() == 0
@pytest.mark.asyncio
async def test_file_group_tag(async_client):
request, response = await async_client.post('/api/files/tag', content=json.dumps(dict(tag_id=1)))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'file_group_id' in response.json['error']
request, response = await async_client.post('/api/files/tag', content=json.dumps(dict(file_group_id=1)))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'tag_id' in response.json['error']
@pytest.mark.asyncio
async def test_search_directories(async_client, test_session, test_directory, make_files_structure,
assert_directories, refresh_files):
"""Directories can be searched by name."""
make_files_structure(['foo/one.txt', 'foo/two.txt', 'bar/one.txt'])
await refresh_files()
assert_directories({'foo', 'bar'})
# Create directory that was not refreshed.
(test_directory / 'baz').mkdir()
# More than one character required.
content = dict(path='f')
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK, response.content
assert response.json['directories'] == []
assert response.json['is_dir'] is False
# Can search using partial directory name.
content = dict(path='fo')
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == ['foo', ]
assert response.json['is_dir'] is False
# Searching directory exactly.
content = dict(path='foo')
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == ['foo', ]
assert response.json['is_dir'] is True
# Case is ignored.
content = dict(path='BAR')
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == ['bar', ]
assert response.json['is_dir'] is False
# Can search directories not yet in DB.
content = dict(path='ba')
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == ['bar', 'baz']
assert response.json['is_dir'] is False
# Searching for something that does not exist.
content = dict(path='does not exist')
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == []
# Only first 20 "foo" are returned.
make_files_structure([
'fooo/',
'foooo/',
'fooooo/',
'foooooo/',
'fooooooo/',
'foooooooo/',
'fooooooooo/',
'foooooooooo/',
'fooooooooooo/',
'foooooooooooo/',
'fooooooooooooo/',
'foooooooooooooo/',
'fooooooooooooooo/',
'foooooooooooooooo/',
'fooooooooooooooooo/',
'foooooooooooooooooo/',
'fooooooooooooooooooo/',
'foooooooooooooooooooo/',
'fooooooooooooooooooooo/',
'foooooooooooooooooooooo/',
'fooooooooooooooooooooooo/',
])
await async_client.post('/api/files/refresh')
content = dict(path='fo')
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == [f'f{"o" * i}' for i in range(2, 22)]
@pytest.mark.asyncio
async def test_post_search_directories(test_session, async_client, make_files_structure):
"""Directory names can be searched. This endpoint also returns Channel and Domain directories."""
channel1_dir, channel2_dir, domain_dir, _ = make_files_structure([
'dir1/',
'dir2/',
'dir3/',
'dir4/',
])
request, response = await async_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
from modules.videos.models import Channel
# Use from_config to create channels properly (creates Collection first)
channel1 = Channel.from_config(test_session, {'directory': channel1_dir, 'name': 'Channel Name'})
channel2 = Channel.from_config(test_session, {'directory': channel2_dir, 'name': 'OtherChannel'})
from wrolpi.collections import Collection
domain_collection = Collection(directory=domain_dir, name='example.com', kind='domain')
test_session.add(domain_collection)
test_session.commit()
# All directories contain "di". The names of the Channel and Directory do not match.
content = {'path': 'di'}
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['directories'] == [{'name': 'dir1', 'path': 'dir1'}, {'name': 'dir2', 'path': 'dir2'},
{'name': 'dir3', 'path': 'dir3'}, {'name': 'dir4', 'path': 'dir4'}]
assert response.json['channel_directories'] == []
assert response.json['domain_directories'] == []
# Channel name matches.
content = {'path': 'Chan'}
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['directories'] == []
assert response.json['channel_directories'] == [
{'name': 'Channel Name', 'path': 'dir1'},
{'name': 'OtherChannel', 'path': 'dir2'},
]
assert response.json['domain_directories'] == []
# "OtherChannel" matches even though the case is wrong, and it has a space.
content = {'path': 'other channel'}
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
# Channel name matches.
assert response.json['directories'] == []
assert response.json['channel_directories'] == [{'name': 'OtherChannel', 'path': 'dir2'}]
assert response.json['domain_directories'] == []
content = {'path': 'exam'}
request, response = await async_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
# Domain name matches.
assert response.json['directories'] == []
assert response.json['channel_directories'] == []
assert response.json['domain_directories'] == [{'domain': 'example.com', 'path': 'dir3'}]
@pytest.mark.asyncio
async def test_post_upload_directory(test_session, async_client, test_directory, make_files_structure,
make_multipart_form):
"""A file can be uploaded in a directory in the destination."""
make_files_structure(['uploads/'])
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='/foo/bar.txt'), # notice the directory
dict(name='totalChunks', value='0'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value='3'),
dict(name='chunk', value='foo', filename='chunk')
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED
assert (test_directory / 'uploads/foo/bar.txt').is_file()
assert (test_directory / 'uploads/foo/bar.txt').read_text() == 'foo'
assert test_session.query(FileGroup).count() == 1
assert test_session.query(Directory).count() == 2
assert {i for i, in test_session.query(Directory.path)} == {
test_directory / 'uploads',
test_directory / 'uploads/foo',
}
@pytest.mark.asyncio
async def test_post_upload(test_session, async_client, test_directory, make_files_structure, make_multipart_form,
tag_factory, video_bytes, test_wrolpi_config):
"""A file can be uploaded in chunks directly to the destination."""
# Disable ffprobe json file creation to avoid unexpected files in test assertions
get_wrolpi_config().save_ffprobe_json = False
make_files_structure(['uploads/'])
tag1, tag2 = await tag_factory(), await tag_factory()
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='video.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=1_000_000),
dict(name='tagNames', value=tag1.name),
dict(name='tagNames', value=tag2.name),
dict(name='chunk', value=part1, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK, response.content.decode()
output = test_directory / 'uploads/video.mp4'
assert output.is_file()
assert output.stat().st_size == 1_000_000
forms = [
dict(name='chunkNumber', value='1'),
dict(name='filename', value='video.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=56318),
dict(name='tagNames', value=tag1.name),
dict(name='tagNames', value=tag2.name),
dict(name='chunk', value=part2, filename='chunk'),
]
body = make_multipart_form(forms)
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
assert output.is_file()
assert get_mimetype(output) == 'video/mp4'
assert output.stat().st_size == len(video_bytes)
assert output.read_bytes() == video_bytes
assert hashlib.md5(output.read_bytes()).hexdigest() == '2738c53bd7c01b01d408da11a55bfa36'
file_group: FileGroup = test_session.query(FileGroup).one()
assert file_group.mimetype == 'video/mp4'
assert set(file_group.tag_names) == {tag1.name, tag2.name}, 'Two tags should be applied.'
assert file_group.indexed, 'File should be indexed after upload.'
assert file_group.model, 'File should be modeled'
# Video was modeled, but has no Channel.
video: Video = test_session.query(Video).one()
assert video.video_path
assert not video.info_json_path
assert not video.channel and not video.channel_id
# Can't upload again because the file already exists.
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='video.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=1_000_000),
dict(name='chunk', value=part1, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.BAD_REQUEST, response.content.decode()
assert response.json['error'] == 'File already exists!'
# FileGroup was not deleted.
file_group: FileGroup = test_session.query(FileGroup).one()
assert file_group
# Remove Tag so file can be uploaded again.
file_group.set_tags(test_session, [])
# An existing file can be overwritten.
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='video.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=1_000_000),
dict(name='chunk', value=part1, filename='chunk'),
dict(name='overwrite', value=True),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK, response.content.decode()
file_group: FileGroup = test_session.query(FileGroup).one_or_none()
assert not file_group, 'Old FileGroup should be deleted when new upload starts.'
# Upload the second part of the file again.
forms = [
dict(name='chunkNumber', value='1'),
dict(name='filename', value='video.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=56318),
dict(name='chunk', value=part2, filename='chunk'),
]
body = make_multipart_form(forms)
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
# Make the same checks as those done above because the file was uploaded again.
assert output.is_file()
assert get_mimetype(output) == 'video/mp4'
assert output.stat().st_size == len(video_bytes)
assert output.read_bytes() == video_bytes
assert hashlib.md5(output.read_bytes()).hexdigest() == '2738c53bd7c01b01d408da11a55bfa36'
file_group: FileGroup = test_session.query(FileGroup).one()
assert file_group.mimetype == 'video/mp4'
assert not file_group.tag_names, 'No tags this time.'
assert file_group.indexed, 'File should be indexed after upload.'
assert file_group.model, 'File should be modeled'
assert isinstance(file_group.get_model_record(test_session), Video)
assert file_group.location == f'/videos/{file_group.id}'
# Video was modeled, but has no Channel.
video: Video = test_session.query(Video).one()
assert video.video_path
assert video.file_group == file_group
assert not video.info_json_path
assert not video.channel and not video.channel_id
@pytest.mark.asyncio
async def test_post_upload_video_extracts_ffprobe(test_session, async_client, test_directory, make_files_structure,
make_multipart_form, video_bytes):
"""Uploading a video file should extract and store ffprobe data."""
make_files_structure(['uploads/'])
# Upload video in chunks (same pattern as test_post_upload)
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
# First chunk
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='test_ffprobe.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=1_000_000),
dict(name='chunk', value=part1, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
# Second chunk (final)
forms = [
dict(name='chunkNumber', value='1'),
dict(name='filename', value='test_ffprobe.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=len(part2)),
dict(name='chunk', value=part2, filename='chunk'),
]
body = make_multipart_form(forms)
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED
# Verify video was created
video: Video = test_session.query(Video).one()
assert video.video_path
# THIS IS THE BUG - ffprobe_json should be populated but it's not
assert video.ffprobe_json is not None, 'ffprobe_json should be extracted on upload'
assert 'streams' in video.ffprobe_json, 'ffprobe_json should contain streams'
assert 'format' in video.ffprobe_json, 'ffprobe_json should contain format'
assert video.file_group.length, 'Duration should be extracted from ffprobe'
assert video.ffprobe_json_path
@pytest.mark.asyncio
async def test_post_upload_text(test_session, async_client, test_directory, make_files_structure, make_multipart_form):
"""A file that cannot be modeled can still be uploaded, and is indexed."""
make_files_structure(['uploads/'])
contents = 'hello'
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='the title.txt'),
dict(name='totalChunks', value='0'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=5),
dict(name='chunk', value=contents, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
file_group: FileGroup = test_session.query(FileGroup).one()
assert file_group.indexed is True
assert file_group.size == 5
assert file_group.a_text == 'the title txt'
assert file_group.d_text == 'hello'
@pytest.mark.asyncio
async def test_post_upload_new_directory(test_session, async_client, test_directory, make_files_structure,
make_multipart_form,
tag_factory, video_bytes):
"""User can create a new directory when uploading."""
tag1, tag2 = await tag_factory(), await tag_factory()
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='video.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='some new directory'),
dict(name='mkdir', value=True),
dict(name='chunkSize', value=len(video_bytes)),
dict(name='tagNames', value=tag1.name),
dict(name='tagNames', value=tag2.name),
dict(name='chunk', value=video_bytes, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK, response.content.decode()
assert (test_directory / 'some new directory').is_dir()
assert (test_directory / 'some new directory/video.mp4').is_file()
assert (test_directory / 'some new directory/video.mp4').read_bytes() == video_bytes
@pytest.mark.asyncio
async def test_directory_crud(test_session, async_client, test_directory, assert_directories, assert_files,
refresh_files):
"""A directory can be created in a subdirectory. Errors are returned if there are conflicts."""
foo = test_directory / 'foo'
foo.mkdir()
await refresh_files()
assert_directories({'foo', })
# Create a subdirectory.
content = dict(path='foo/bar')
request, response = await async_client.post('/api/files/directory', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert (test_directory / 'foo/bar').is_dir()
assert_directories({'foo', 'foo/bar'})
# Cannot create twice.
content = dict(path='foo/bar')
request, response = await async_client.post('/api/files/directory', content=json.dumps(content))
assert response.status_code == HTTPStatus.CONFLICT
assert (test_directory / 'foo/bar').is_dir()
assert_directories({'foo', 'foo/bar'})
# Create a file to be deleted.
(test_directory / 'foo/bar/asdf.txt').write_text('asdf')
# Can get information about a directory.
request, response = await async_client.post('/api/files/get_directory', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['path'] == 'foo/bar'
assert response.json['size'] == 4
assert response.json['file_count'] == 1
# Can rename the directory.
content = dict(path='foo/bar', new_name='baz')
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not (test_directory / 'foo/bar').exists()
assert (test_directory / 'foo/baz').exists()
assert (test_directory / 'foo/baz/asdf.txt').exists()
# Deletion is recursive.
assert (test_directory / 'foo/baz/asdf.txt').is_file()
content = dict(paths=['foo/baz', ])
request, response = await async_client.post('/api/files/delete', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not (test_directory / 'foo/baz').exists()
assert not (test_directory / 'foo/baz/asdf.txt').exists()
# Only top directory is left.
assert_directories({'foo', })
@pytest.mark.asyncio
async def test_move(test_session, test_directory, make_files_structure, async_client, await_background_tasks):
"""Files can be moved up and down the media directory. Destination directories should already exist."""
baz, bar = make_files_structure({
'foo/bar.txt': 'bar',
'baz.txt': 'baz',
})
# qux directory does not exist
content = dict(paths=['foo', 'baz.txt'], destination='qux')
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NOT_FOUND
(test_directory / 'qux').mkdir()
(test_directory / 'quux/quuz').mkdir(parents=True)
# mv foo baz.txt qux
content = dict(paths=['foo', 'baz.txt'], destination='qux')
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert not bar.exists()
assert (test_directory / 'qux/foo/bar.txt').is_file()
# mv foo/bar.txt qux
content = dict(paths=['qux/foo/bar.txt', ], destination='qux')
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert not bar.exists()
assert (test_directory / 'qux/bar.txt').is_file()
assert (test_directory / 'qux/foo').is_dir()
# mv qux/bar.txt ./
content = dict(paths=['qux/bar.txt', ], destination='')
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert (test_directory / 'bar.txt').is_file()
# mv bar.txt quux/quuz/bar.txt
content = dict(paths=['bar.txt', ], destination='quux/quuz')
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert (test_directory / 'quux/quuz/bar.txt').is_file()
# mv quux/quuz/bar.txt quux/bar.txt
content = dict(paths=['quux/quuz/bar.txt', ], destination='quux')
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert (test_directory / 'quux/bar.txt').is_file()
assert (test_directory / 'qux/foo').is_dir()
assert (test_directory / 'quux/quuz').is_dir()
@pytest.mark.asyncio
async def test_rename_file(test_session, test_directory, make_files_structure, async_client, refresh_files):
"""A FileGroup can be renamed. The title and search index is updated."""
foo_file, = make_files_structure({
'foo/bar/baz.txt': 'asdf',
})
await refresh_files()
foo_fg = test_session.query(FileGroup).one()
assert foo_fg.a_text == 'baz txt'
assert foo_fg.d_text == 'asdf'
# mv foo/bar/baz.txt foo/bar/qux.txt
content = dict(path='foo/bar/baz.txt', new_name='qux.txt')
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not foo_file.exists()
assert (test_directory / 'foo/bar/qux.txt').is_file()
assert (test_directory / 'foo/bar/qux.txt').read_text() == 'asdf'
assert foo_fg.a_text == 'qux txt'
assert foo_fg.d_text == 'asdf'
@pytest.mark.asyncio
async def test_rename_file_with_associated_files_via_api(test_session, test_directory, make_files_structure,
async_client, video_bytes, srt_text, refresh_files):
"""Renaming a FileGroup via API should rename all associated files.
Regression test: When renaming "example.mp4" to "example 2.mp4" and back via API,
associated files like "example.srt" should also be renamed both times.
"""
video, srt = make_files_structure({
'example.mp4': video_bytes,
'example.srt': srt_text,
})
await refresh_files()
# Verify initial state
fg = test_session.query(FileGroup).one()
assert fg.primary_path == video
# Video validation creates .ffprobe.json, so we have 3 files
assert len(fg.files) == 3
# First rename via API: "example.mp4" -> "example 2.mp4"
content = dict(path='example.mp4', new_name='example 2.mp4')
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Verify first rename - both files should be renamed
new_video_1 = test_directory / 'example 2.mp4'
new_srt_1 = test_directory / 'example 2.srt'
assert new_video_1.is_file(), "Video should be renamed to 'example 2.mp4'"
assert new_srt_1.is_file(), "SRT should be renamed to 'example 2.srt'"
assert not video.exists(), "Old video should not exist"
assert not srt.exists(), "Old SRT should not exist"
# Verify FileGroup is updated
fg = test_session.query(FileGroup).one()
assert fg.primary_path == new_video_1
file_names = [f['path'] for f in fg.files]
assert 'example 2.mp4' in file_names
assert 'example 2.srt' in file_names
assert len(fg.files) == 3
# Second rename via API: "example 2.mp4" -> "example.mp4"
content = dict(path='example 2.mp4', new_name='example.mp4')
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Verify second rename - BOTH files should be renamed back
final_video = test_directory / 'example.mp4'
final_srt = test_directory / 'example.srt'
assert final_video.is_file(), "Video should be renamed back to 'example.mp4'"
assert final_srt.is_file(), "SRT should be renamed back to 'example.srt'"
assert not new_video_1.exists(), "'example 2.mp4' should not exist"
assert not new_srt_1.exists(), "'example 2.srt' should not exist"
# Verify FileGroup is updated
fg = test_session.query(FileGroup).one()
assert fg.primary_path == final_video
file_names = [f['path'] for f in fg.files]
assert 'example.mp4' in file_names
assert 'example.srt' in file_names
assert len(fg.files) == 3
@pytest.mark.asyncio
async def test_rename_directory(test_session, test_directory, make_files_structure, async_client, refresh_files):
make_files_structure({
'foo/bar/baz.txt': 'asdf',
})
await refresh_files()
bar, foo = test_session.query(Directory).order_by(Directory.name).all()
assert foo.name == 'foo' and bar.name == 'bar'
# mv foo/bar foo/qux
content = dict(path='foo/bar', new_name='qux')
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not (test_directory / 'foo/bar').exists()
assert (test_directory / 'foo/qux/baz.txt').is_file()
assert (test_directory / 'foo/qux/baz.txt').read_text() == 'asdf'
foo, qux = test_session.query(Directory).order_by(Directory.name).all()
assert foo.name == 'foo' and qux.name == 'qux'
@pytest.mark.asyncio
async def test_rename_directory_stale_destination_fails_promptly(
test_session, test_directory, make_files_structure, async_client, refresh_files
):
"""A directory rename that collides with a stale FileGroup must fail promptly.
Regression: the move's IntegrityError left the job 'pending', so
POST /api/files/rename hung until wait_for_job's 300s timeout.
"""
import asyncio
import shutil
make_files_structure({'dir-a/one.txt': 'hello'})
await refresh_files()
content = dict(path='dir-a', new_name='dir-b')
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Remove dest on disk without refreshing; recreate the old source.
shutil.rmtree(test_directory / 'dir-b')
(test_directory / 'dir-a').mkdir()
(test_directory / 'dir-a' / 'one.txt').write_text('hello')
request, response = await asyncio.wait_for(
async_client.post('/api/files/rename', content=json.dumps(content)),
timeout=5,
)
assert response.status_code == HTTPStatus.CONFLICT, response.json
@pytest.mark.asyncio
async def test_rename_collection_directory(test_session, test_directory, make_files_structure, async_client,
refresh_files):
"""Renaming a Collection's directory updates the Collection.directory."""
from wrolpi.collections.models import Collection
# Create a collection with a directory
collection_dir = test_directory / 'my_collection'
collection_dir.mkdir()
(collection_dir / 'file.txt').write_text('contents')
collection = Collection(name='Test Collection', kind='channel', directory=collection_dir)
test_session.add(collection)
test_session.commit()
collection_id = collection.id
await refresh_files()
# Rename the directory
content = dict(path='my_collection', new_name='renamed_collection')
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Verify Collection.directory was updated
collection = test_session.query(Collection).filter_by(id=collection_id).one()
assert collection.directory == test_directory / 'renamed_collection'
assert not (test_directory / 'my_collection').exists()
assert (test_directory / 'renamed_collection' / 'file.txt').is_file()
@pytest.mark.asyncio
async def test_rename_directory_with_nested_collections(test_session, test_directory, async_client, refresh_files):
"""Renaming a parent directory updates nested Collection directories and download destinations."""
from wrolpi.collections.models import Collection
from wrolpi.downloader import Download
# Create parent directory with nested collection directories
parent_dir = test_directory / 'videos'
parent_dir.mkdir()
channel_a_dir = parent_dir / 'channel_a'
channel_a_dir.mkdir()
(channel_a_dir / 'video1.mp4').write_bytes(b'fake video 1')
channel_b_dir = parent_dir / 'channel_b'
channel_b_dir.mkdir()
(channel_b_dir / 'video2.mp4').write_bytes(b'fake video 2')
# Create two channel collections for the nested directories
coll_a = Collection(name='Channel A', kind='channel', directory=channel_a_dir)
coll_b = Collection(name='Channel B', kind='channel', directory=channel_b_dir)
test_session.add_all([coll_a, coll_b])
test_session.commit()
# Create a download associated with channel_a
download = Download(url='https://example.com/channel_a', destination=channel_a_dir, collection_id=coll_a.id)
test_session.add(download)
test_session.commit()
coll_a_id, coll_b_id, download_id = coll_a.id, coll_b.id, download.id
await refresh_files()
# Rename the parent directory
content = dict(path='videos', new_name='my_videos')
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Verify nested Collection directories were updated
coll_a = test_session.query(Collection).filter_by(id=coll_a_id).one()
coll_b = test_session.query(Collection).filter_by(id=coll_b_id).one()
assert coll_a.directory == test_directory / 'my_videos' / 'channel_a'
assert coll_b.directory == test_directory / 'my_videos' / 'channel_b'
# Verify download destination was updated
download = test_session.query(Download).filter_by(id=download_id).one()
assert download.destination == test_directory / 'my_videos' / 'channel_a'
# Verify files were moved
assert (test_directory / 'my_videos' / 'channel_a' / 'video1.mp4').exists()
assert (test_directory / 'my_videos' / 'channel_b' / 'video2.mp4').exists()
assert not (test_directory / 'videos').exists()
@pytest.mark.asyncio
async def test_delete_directory_recursive(test_session, test_directory, make_files_structure, async_client,
assert_files, refresh_files):
make_files_structure(['dir/foo', 'dir/bar', 'empty'])
await refresh_files()
assert test_session.query(FileGroup).count() == 3
content = {'paths': ['dir/', ]}
request, response = await async_client.post('/api/files/delete', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(FileGroup).count() == 1
empty = test_session.query(FileGroup).one()
assert empty.primary_path.name == 'empty'
@pytest.mark.asyncio
async def test_get_file(test_session, async_client, test_directory, make_files_structure, refresh_files):
"""Can get info about a single file."""
make_files_structure({'foo/bar.txt': 'foo contents'})
await refresh_files()
content = dict(file='foo/bar.txt')
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file']
assert response.json['file']['path'] == 'foo/bar.txt'
fg, = test_session.query(FileGroup).all()
assert fg.primary_path == test_directory / 'foo/bar.txt'
assert isinstance(fg.viewed, datetime)
@pytest.mark.asyncio
async def test_ignore_directory(test_session, async_client, test_directory, make_files_structure, test_wrolpi_config,
await_switches, refresh_files):
"""A maintainer can ignore/un-ignore directories. The files in the directory should not be refreshed."""
# Remove any default ignored directories.
get_wrolpi_config().ignored_directories = []
foo, bar, baz = make_files_structure(['foo/foo.txt', 'foo/bar.txt', 'baz/baz.txt'])
assert len(get_wrolpi_config().ignored_directories) == 0
# Ignore baz/
content = dict(path=str(baz.parent))
request, response = await async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.OK
assert len(get_wrolpi_config().ignored_directories) == 1
await refresh_files()
files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'wrolpi.yaml'}
# Un-ignore baz/
request, response = await async_client.post('/api/files/unignore_directory', json=content)
assert response.status_code == HTTPStatus.OK
assert len(get_wrolpi_config().ignored_directories) == 0
await refresh_files()
files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
# Ignore configs.
files = {i for i in files if not i.primary_path.name.endswith('.yaml')}
assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'baz.txt'}
# Cannot ignore special directories.
content = dict(path=str(test_directory / 'videos'))
request, response = await async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert len(get_wrolpi_config().ignored_directories) == 0
# Zims directory depends on FileGroup indexing for Manage/search — cannot ignore.
content = dict(path=str(test_directory / 'zims'))
request, response = await async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert len(get_wrolpi_config().ignored_directories) == 0
# Relative form of zims is also refused.
content = dict(path='zims')
request, response = await async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert len(get_wrolpi_config().ignored_directories) == 0
# Map is allowed to be ignored (lists PMTiles from the filesystem, not FileGroups).
content = dict(path=str(test_directory / 'map'))
request, response = await async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.OK
assert any(pathlib.Path(i).name == 'map' or str(i).endswith('/map')
for i in get_wrolpi_config().ignored_directories)
@pytest.mark.asyncio
async def test_sanitize_ignored_zims_directory(test_session, async_client, test_directory, test_wrolpi_config):
"""Zims previously saved as ignored is stripped so refresh can index them again."""
from wrolpi.files.lib import (
get_normalized_ignored_directories,
sanitize_ignored_directories,
)
# Simulate an older config that incorrectly ignored zims (and intentionally ignored map).
get_wrolpi_config().ignored_directories = ['config', 'tags', 'map', 'zims']
# Normalized list used by refresh must never exclude zims.
normalized = get_normalized_ignored_directories()
assert not any(pathlib.Path(i).name == 'zims' for i in normalized)
assert any(pathlib.Path(i).name == 'map' for i in normalized)
sanitize_ignored_directories()
assert 'zims' not in get_wrolpi_config().ignored_directories
assert 'map' in get_wrolpi_config().ignored_directories
# Settings endpoint also sanitizes so the UI eye-icon matches.
request, response = await async_client.get('/api/settings')
assert response.status_code == HTTPStatus.OK
ignored = [str(i) for i in response.json['ignored_directories']]
assert 'zims' not in ignored
assert 'map' in ignored
# --- Bulk Tagging API Tests ---
@pytest.mark.asyncio
async def test_bulk_tag_preview_api(test_session, async_client, make_files_structure, tag_factory):
"""The bulk tag preview endpoint returns file count and shared tags."""
foo, bar, baz = make_files_structure({
'foo.txt': 'foo',
'bar.txt': 'bar',
'baz.txt': 'baz',
})
# Create FileGroups
fg_foo = FileGroup.from_paths(test_session, foo)
fg_bar = FileGroup.from_paths(test_session, bar)
fg_baz = FileGroup.from_paths(test_session, baz)
test_session.commit()
# Add shared and non-shared tags
tag1 = await tag_factory('shared_tag')
tag2 = await tag_factory('only_foo')
fg_foo.add_tag(test_session, tag1.id)
fg_foo.add_tag(test_session, tag2.id)
fg_bar.add_tag(test_session, tag1.id)
fg_baz.add_tag(test_session, tag1.id)
test_session.commit()
content = {'paths': ['foo.txt', 'bar.txt', 'baz.txt']}
request, response = await async_client.post('/api/files/bulk_tag/preview', json=content)
assert response.status_code == HTTPStatus.OK
assert response.json['file_count'] == 3
assert 'shared_tag' in response.json['shared_tag_names']
assert 'only_foo' not in response.json['shared_tag_names']
@pytest.mark.asyncio
async def test_bulk_tag_preview_empty(test_session, async_client):
"""The bulk tag preview endpoint handles empty paths."""
content = {'paths': []}
request, response = await async_client.post('/api/files/bulk_tag/preview', json=content)
assert response.status_code == HTTPStatus.OK
assert response.json['file_count'] == 0
assert response.json['shared_tag_names'] == []
@pytest.mark.asyncio
async def test_bulk_tag_apply_api(test_session, async_client, make_files_structure, tag_factory):
"""The bulk tag apply endpoint queues a tagging job."""
foo, bar = make_files_structure({
'foo.txt': 'foo',
'bar.txt': 'bar',
})
# Create FileGroups
FileGroup.from_paths(test_session, foo)
FileGroup.from_paths(test_session, bar)
test_session.commit()
# Create a tag
tag = await tag_factory('api_tag')
test_session.commit()
content = {
'paths': ['foo.txt', 'bar.txt'],
'add_tag_names': ['api_tag'],
'remove_tag_names': [],
}
request, response = await async_client.post('/api/files/bulk_tag/apply', json=content)
assert response.status_code == HTTPStatus.ACCEPTED
@pytest.mark.asyncio
async def test_bulk_tag_progress_api(test_session, async_client):
"""The bulk tag progress endpoint returns progress information."""
request, response = await async_client.get('/api/files/bulk_tag/progress')
assert response.status_code == HTTPStatus.OK
assert 'status' in response.json
assert 'total' in response.json
assert 'completed' in response.json
assert 'queued_jobs' in response.json
# --- File Tracking Tests ---
@pytest.mark.asyncio
async def test_get_file_skip_tracking(test_session, async_client, test_directory, make_files_structure, refresh_files):
"""Can get info about a single file without tracking the view."""
make_files_structure({'foo/bar.txt': 'foo contents'})
await refresh_files()
# Request the file with skip_tracking=True
content = dict(file='foo/bar.txt', skip_tracking=True)
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file']
assert response.json['file']['path'] == 'foo/bar.txt'
# FileGroup.viewed should not be set
fg, = test_session.query(FileGroup).all()
assert fg.primary_path == test_directory / 'foo/bar.txt'
assert fg.viewed is None, "FileGroup.viewed should not be set when skip_tracking=True"
@pytest.mark.asyncio
async def test_get_file_ignored_directory_not_tracked(test_session, async_client, test_directory, make_files_structure,
refresh_files, test_wrolpi_config, await_switches):
"""Files in ignored directories do not get their viewed timestamp set."""
# Create files in a directory that will be ignored
foo, bar = make_files_structure({'config/settings.txt': 'settings', 'normal/file.txt': 'contents'})
# Ignore the config directory
get_wrolpi_config().ignored_directories = [str(test_directory / 'config')]
await await_switches()
# Refresh only the normal directory (config is ignored)
await refresh_files()
# The normal file should be in the database
normal_fg = test_session.query(FileGroup).filter(FileGroup.primary_path.like('%normal%')).one()
assert normal_fg.viewed is None
# Request the normal file - it should be tracked
content = dict(file='normal/file.txt')
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
test_session.refresh(normal_fg)
assert normal_fg.viewed is not None, "Normal files should be tracked"
# Request a file in the ignored directory - it should return the file data
# but should not be tracked (since it's in an ignored directory)
content = dict(file='config/settings.txt')
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
# Verify the ignored file is not in the database (because it was never refreshed)
ignored_fgs = test_session.query(FileGroup).filter(FileGroup.primary_path.like('%config%')).all()
assert len(ignored_fgs) == 0, "Files in ignored directories should not be in the database"