rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
wrolpi/modules/videos/test/test_video.py
lrnselfreliance cfbed9e4d7
db: serializers read only loaded data (#777) 
* db: serializers read only loaded data

Serializing a model must not query.  A lazy load there is a hidden query per row (search serialized 24 rows
with 55 statements), and would be a `MissingGreenlet` error under an async Session.

- `wrolpi.db.no_db_access(reason)` and `@serializer` (on every model `__json__`, and around `json_response`):
  an engine `before_cursor_execute` listener raises `UnexpectedQuery` during tests, and in production logs once
  per serializer and lets the query run.  Tests record each trip as well, and the autouse
  `no_unexpected_queries` fixture fails a test even when the code caught the error (Video's serializer was
  swallowing its per-row loads of the deferred `ffprobe_json`).
- Relationships serializers read load with their parent: `selectin` for FileGroup.tag_files and
  Collection.downloads; `joined` for TagFile.tag, Collection.tag, Channel.collection, Video.channel,
  Archive.collection, ZimSubscription.download, CollectionItem.zim, and the file_group of Video, Archive, Doc,
  Zim and CollectionItem.
- Search and the Video lookups the video page uses load the deferred `ffprobe_json` (`undefer`).
- `wrolpi/test/test_serializers.py`: every serialized model, freshly loaded, serializes without SQL.

Search: 55 -> 5 statements for 24 videos (31 -> 5 for archives), and a few ms faster locally.

Not in this change: guarding the `dict()` serializers and `expire_on_commit=False`, which go together (about 50
serialize-after-commit sites, and 40 tests that read values changed elsewhere after a commit).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* db: serializer loads per query; the guard ignores transaction control

The query guard reported the engine's own BEGIN (emitted through the same event as the first statement on a
Session with no open transaction) instead of the lazy load after it, and in production that BEGIN used up the
one warning per serializer.  Transaction-control statements are now ignored.

Mapper-level eager loads reached background code: every Video/Archive load in the modelers (one per file,
inside the write transaction) joined file_group/channel/collection/tag and selectin-loaded the Collection's
Download rows with their yt-dlp info_json.  A second refresh of 100 videos took 0.40 s instead of 0.16 s.

Serializer needs are now declared per model (`json_options()`) and added only by the queries whose results
are serialized; Download.info_json is deferred there (no serializer reads it).  Mapper defaults keep only
narrow joins nearly every use needs: TagFile.tag, Collection.tag and Channel.collection.  A second refresh is
back to master's statement count and time; search still drops from 55 to 3 statements.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 18:28:08 -06:00

332 lines
14 KiB
Python
Raw Blame History

import json
from datetime import datetime, timezone
from http import HTTPStatus
import pytest
from modules.videos.models import Video
from wrolpi.errors import FileGroupIsTagged
from wrolpi.files.models import FileGroup
def test_delete_video_no_channel(test_session, simple_video):
"""A Video can be deleted even if it does not have a channel."""
assert simple_video.channel
simple_video.channel_id = None
test_session.commit()
paths = list(simple_video.file_group.my_paths())
assert not simple_video.channel
simple_video.delete()
# All files were deleted.
for path in paths:
assert not path.exists()
test_session.commit()
assert test_session.query(FileGroup).count() == 0
assert test_session.query(Video).count() == 0
@pytest.mark.asyncio
async def test_delete_video_with_tag(async_client, test_session, video_factory, tag_factory):
"""You cannot delete a video with a tag."""
video = video_factory(with_video_file=True, with_poster_ext='png')
tag = await tag_factory()
video.add_tag(test_session, tag.name)
test_session.commit()
with pytest.raises(FileGroupIsTagged):
video.delete()
# Video was not deleted.
test_session.commit()
assert test_session.query(Video).count() == 1
assert test_session.query(FileGroup).count() == 1
@pytest.mark.asyncio
async def test_delete_video_api_tagged_requires_force(async_client, test_session, video_factory, tag_factory,
await_switches):
"""The unified delete endpoint returns 409 for tagged videos until force=true is passed."""
video = video_factory(with_video_file=True, with_poster_ext='png')
tag = await tag_factory()
video.add_tag(test_session, tag.name)
test_session.commit()
file_group_id = video.file_group_id
paths = list(video.file_group.my_paths())
body = json.dumps({'file_group_ids': [file_group_id]})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.CONFLICT
assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
assert len(response.json['file_groups']) == 1
assert response.json['file_groups'][0]['id'] == file_group_id
assert tag.name in response.json['file_groups'][0]['tags']
# Nothing was deleted.
assert all(p.is_file() for p in paths)
assert test_session.query(Video).count() == 1
# With force=true, the Video is deleted.
body = json.dumps({'file_group_ids': [file_group_id], 'force': True})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.NO_CONTENT
assert all(not p.exists() for p in paths)
assert test_session.query(Video).count() == 0
assert test_session.query(FileGroup).count() == 0
await await_switches()
@pytest.mark.asyncio
async def test_delete_video_api_atomic_when_some_tagged(async_client, test_session, video_factory, tag_factory):
"""When deleting multiple Videos and any are tagged, NONE are deleted until force=true."""
untagged = video_factory(with_video_file=True)
tagged = video_factory(with_video_file=True)
tag = await tag_factory()
tagged.add_tag(test_session, tag.name)
test_session.commit()
untagged_paths = list(untagged.file_group.my_paths())
tagged_paths = list(tagged.file_group.my_paths())
body = json.dumps({'file_group_ids': [untagged.file_group_id, tagged.file_group_id]})
request, response = await async_client.post('/api/files/delete_groups', content=body)
assert response.status_code == HTTPStatus.CONFLICT
assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
returned_ids = {fg['id'] for fg in response.json['file_groups']}
assert returned_ids == {tagged.file_group_id}
# Nothing was deleted.
assert all(p.is_file() for p in untagged_paths)
assert all(p.is_file() for p in tagged_paths)
assert test_session.query(Video).count() == 2
@pytest.mark.asyncio
async def test_video_channel_refresh(async_client, test_session, test_directory, channel_factory, video_factory,
refresh_files):
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
# Create a video file in this channel's directory.
channel = channel_factory()
video1_path = channel.directory / 'video1.mp4'
video_factory(channel_id=channel.id, with_video_file=video1_path, with_poster_ext='jpg')
# Create another video outside the channel's directory. It should not have a channel.
video2_path = test_directory / 'video2.mp4'
video_factory(with_video_file=video2_path)
# Delete all FileGroups so the channel will be associated again.
test_session.query(FileGroup).delete()
test_session.commit()
await refresh_files()
assert test_session.query(FileGroup).count() == 2
assert test_session.query(Video).count() == 2
# The video is associated with `simple_channel` because its file is in the Channel's directory.
video1: Video = Video.get_by_path(test_session, video1_path)
video2: Video = Video.get_by_path(test_session, video2_path)
assert video1.channel == channel
assert not video2.channel
# Serializers do not query: load what Video's `__json__` reads, as the queries that serialize Videos do.
video1 = test_session.query(Video).options(*Video.json_options()).populate_existing() \
.filter(Video.id == video1.id).one()
assert video1.__json__()['video']['channel']
video_channel = video1.__json__()['video']['channel']
assert video_channel['id'] == channel.id
assert video_channel['name'] == channel.name
@pytest.mark.asyncio
async def test_delete_duplicate_video(async_client, test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, and everything about the files matches, delete a random one."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]}
entry = channel.info_json['entries'][0]
tag = await tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
tag_names=[tag.name, ]
)
vid2 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title ', # Video was renamed, trailing space was removed.
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
)
vid1.file_group.url = vid2.file_group.url = 'https://example.com/video'
assert test_session.query(Video).count() == 2
test_session.commit()
assert await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
test_session.commit()
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
video = test_session.query(Video).one()
assert video.video_path == video_path, 'Video path does not match'
assert video.video_path.is_file()
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
assert video.file_group.tag_names == [tag.name, ]
# Running again has no effect.
assert not await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
test_session.commit()
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
video = test_session.query(Video).one()
assert video.video_path.is_file()
assert video.video_path == video_path, 'Video path does not match'
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
assert video.file_group.tag_names == [tag.name, ]
@pytest.mark.asyncio
async def test_delete_duplicate_video_tagged_duplicate(async_client, test_session, channel_factory, video_factory,
tag_factory):
"""Deleting a duplicate Video should work even when the duplicate being deleted has tags."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]}
entry = channel.info_json['entries'][0]
tag = await tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
)
vid2 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title ',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
tag_names=[tag.name, ], # The duplicate being deleted has a tag.
)
vid1.file_group.url = vid2.file_group.url = 'https://example.com/video'
assert test_session.query(Video).count() == 2
test_session.commit()
assert await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
test_session.commit()
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
video = test_session.query(Video).one()
assert video.video_path == video_path
assert video.video_path.is_file()
# The tag from the deleted duplicate should be consolidated onto the kept video.
assert video.file_group.tag_names == [tag.name, ]
@pytest.mark.asyncio
async def test_delete_duplicate_video_smaller_copy_at_destination(async_client, test_session, channel_factory,
video_factory):
"""The kept Video can be renamed onto the path of the smaller duplicate that was just deleted."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]}
entry = channel.info_json['entries'][0]
# The smaller copy already occupies the destination path.
small = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
)
large = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title ', # Trailing space, so a different path.
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
)
small.file_group.url = large.file_group.url = 'https://example.com/video'
large.file_group.size = small.file_group.size + 1
large_id = large.id
large_old_path = large.video_path
test_session.commit()
assert small.video_path == video_path
assert await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
# The commit must not raise IntegrityError: the deleted duplicate still held `video_path` when the kept Video's
# `primary_path` was updated to it.
test_session.commit()
video = test_session.query(Video).one()
assert video.id == large_id, 'The larger Video was not the one kept'
assert video.video_path == video_path
assert video.video_path.is_file()
assert not large_old_path.exists(), 'The kept Video was copied to the destination rather than moved'
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
@pytest.mark.asyncio
async def test_delete_renamed_video(async_client, test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, delete all the videos that do not have the new title."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4'
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'Some new title'}]}
entry = channel.info_json['entries'][0]
tag = await tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
tag_names=[tag.name, ]
)
vid2 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title ', # Video was renamed, trailing space was removed.
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
)
vid1.file_group.url = vid2.file_group.url = 'https://example.com/video'
assert test_session.query(Video).count() == 2
test_session.commit()
await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
test_session.commit()
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
video = test_session.query(Video).one()
assert video.video_path == video_path, 'Video path does not match'
assert video.video_path.is_file()
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
assert video.file_group.tag_names == [tag.name, ]
Reference in a new issue View git blame Copy permalink