mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 23:46:18 -05:00
Fixing bug where Archive screenshots would not be tracked when a user uploads a screenshot.
Some checks failed
CodeQL / Analyze (push) Has been cancelled
Some checks failed
CodeQL / Analyze (push) Has been cancelled
This commit is contained in:
parent
ecac129816
commit
c635267eda
13 changed files with 368 additions and 60 deletions
|
|
@ -181,8 +181,12 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
|||
if readability_txt_path:
|
||||
contents = get_article(readability_txt_path)
|
||||
|
||||
archive = Archive(file_group_id=file_group_id, file_group=file_group)
|
||||
session.add(archive)
|
||||
# Check if an Archive already exists for this FileGroup
|
||||
archive = session.query(Archive).filter_by(file_group_id=file_group_id).one_or_none()
|
||||
if not archive:
|
||||
# Create new Archive if it doesn't exist
|
||||
archive = Archive(file_group_id=file_group_id, file_group=file_group)
|
||||
session.add(archive)
|
||||
archive.validate()
|
||||
archive.flush()
|
||||
|
||||
|
|
@ -196,6 +200,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
|||
'readability_txt_path': archive.readability_txt_path,
|
||||
'screenshot_path': archive.screenshot_path,
|
||||
'singlefile_path': archive.singlefile_path,
|
||||
'info_json_path': archive.info_json_path,
|
||||
}
|
||||
|
||||
return archive
|
||||
|
|
|
|||
|
|
@ -104,7 +104,11 @@ async def singlefile_upload_switch_handler(url=None):
|
|||
logger.trace(f'singlefile_upload_switch_handler called on empty queue')
|
||||
return
|
||||
|
||||
q_size = q.qsize()
|
||||
try:
|
||||
q_size = q.qsize()
|
||||
except NotImplementedError:
|
||||
# qsize() is not implemented on macOS
|
||||
q_size = '?'
|
||||
logger.info(f'singlefile_upload_switch_handler queue size: {q_size}')
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ def archive_directory(test_directory) -> pathlib.Path:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def archive_factory(test_session, archive_directory, make_files_structure):
|
||||
def archive_factory(test_session, archive_directory, make_files_structure, image_bytes_factory):
|
||||
def time_generator():
|
||||
timestamp = datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC)
|
||||
while True:
|
||||
|
|
@ -29,7 +29,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
now = time_generator()
|
||||
|
||||
def _(domain: str = None, url: str = None, title: str = 'NA', contents: str = None,
|
||||
singlefile_contents: str = None, tag_names: List[str] = None) -> Archive:
|
||||
singlefile_contents: str = None, tag_names: List[str] = None, screenshot: bool = True) -> Archive:
|
||||
if domain:
|
||||
assert '/' not in domain
|
||||
|
||||
|
|
@ -49,10 +49,9 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
if url:
|
||||
json_contents['url'] = url
|
||||
json_contents = json.dumps(json_contents)
|
||||
singlefile_path, screenshot_path, readability_path, readability_txt_path, readability_json_path = \
|
||||
singlefile_path, readability_path, readability_txt_path, readability_json_path = \
|
||||
make_files_structure({
|
||||
str(domain_dir / f'{timestamp}_{title}.html'): singlefile_contents or '<html></html>',
|
||||
str(domain_dir / f'{timestamp}_{title}.png'): None,
|
||||
str(domain_dir / f'{timestamp}_{title}.readability.html'): '<html></html>',
|
||||
str(domain_dir / f'{timestamp}_{title}.readability.txt'): contents,
|
||||
str(domain_dir / f'{timestamp}_{title}.readability.json'): json_contents,
|
||||
|
|
@ -61,6 +60,11 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
if domain:
|
||||
domain = test_session.query(Domain).filter_by(domain=domain).one_or_none()
|
||||
|
||||
screenshot_path = None
|
||||
if screenshot:
|
||||
screenshot_path = domain_dir / f'{timestamp}_{title}.png'
|
||||
screenshot_path.write_bytes(image_bytes_factory())
|
||||
|
||||
if not domain and domain_dir.name != 'NA':
|
||||
domain = Domain(
|
||||
domain=domain_dir.name,
|
||||
|
|
|
|||
|
|
@ -197,6 +197,18 @@ class Archive(Base, ModelHelper):
|
|||
if screenshot_file := self.screenshot_file:
|
||||
return screenshot_file['path']
|
||||
|
||||
@property
|
||||
def info_json_file(self) -> Optional[dict]:
|
||||
files = self.file_group.my_files()
|
||||
for file in files:
|
||||
if file['path'].name.endswith('.info.json'):
|
||||
return file
|
||||
|
||||
@property
|
||||
def info_json_path(self) -> Optional[pathlib.Path]:
|
||||
if info_json_file := self.info_json_file:
|
||||
return info_json_file['path']
|
||||
|
||||
def apply_readability_data(self):
|
||||
"""Read the Readability JSON file, apply its contents to this record."""
|
||||
readability_json_path = self.readability_json_path
|
||||
|
|
|
|||
|
|
@ -4,8 +4,11 @@ from http import HTTPStatus
|
|||
import pytest
|
||||
|
||||
from modules.archive import lib, Archive
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.test.common import skip_circleci, skip_macos
|
||||
|
||||
from wrolpi.common import get_relative_to_media_directory
|
||||
|
||||
|
||||
def check_results(test_client, data, ids):
|
||||
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
|
||||
|
|
@ -202,7 +205,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
|
|||
request, response = test_client.get(f'/api/archive/domains')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['domains'][0]['domain'] == 'example.com'
|
||||
assert response.json['domains'][0]['size'] == 116
|
||||
assert response.json['domains'][0]['size'] == 254
|
||||
|
||||
# Deleting works.
|
||||
request, response = test_client.delete(f'/api/archive/{archive1.id}')
|
||||
|
|
@ -257,3 +260,107 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa
|
|||
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
|
||||
events = list(events_history)
|
||||
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_upload_file_tracking(test_session, async_client, archive_directory, archive_factory,
|
||||
await_switches, make_multipart_form, image_bytes_factory):
|
||||
"""Test that uploading an info.json and image file via /api/files/upload properly tracks them in Archive and FileGroup."""
|
||||
archive = archive_factory('example.com', 'https://example.com/test-url', 'tracking test', screenshot=False)
|
||||
test_session.commit()
|
||||
|
||||
# Verify Archive and FileGroup are created, but screenshot does not exist.
|
||||
file_group = archive.file_group
|
||||
assert not archive.screenshot_path, 'Screenshot should not exist yet.'
|
||||
assert file_group.files is not None, 'FileGroup should have files array'
|
||||
initial_file_count = len(file_group.files)
|
||||
|
||||
# Upload info.json via /api/files/upload
|
||||
# Extract the prefix from singlefile path (e.g., "2000-01-01-00-00-01_tracking test")
|
||||
singlefile_stem = archive.singlefile_path.stem # e.g., "2000-01-01-00-00-01_tracking test"
|
||||
|
||||
# Create info.json content
|
||||
info_json_data = {
|
||||
'title': 'test',
|
||||
'url': 'https://example.com/test-url',
|
||||
'custom_field': 'value'
|
||||
}
|
||||
info_json_bytes = json.dumps(info_json_data).encode()
|
||||
|
||||
# Get relative destination directory
|
||||
destination = str(get_relative_to_media_directory(archive.singlefile_path.parent))
|
||||
filename = f'{singlefile_stem}.info.json'
|
||||
|
||||
# POST to /api/files/upload
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(info_json_bytes))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=info_json_bytes, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Step 4: Verify info.json is tracked
|
||||
# Refresh the session to get updated data
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
|
||||
# Assert info.json properties exist
|
||||
assert archive.info_json_path is not None, 'Archive.info_json_path should exist'
|
||||
assert archive.info_json_file is not None, 'Archive.info_json_file should exist'
|
||||
|
||||
# Assert FileGroup has the correct number of files.
|
||||
assert len(archive.file_group.files) == initial_file_count + 1, \
|
||||
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
|
||||
assert archive.file_group.data.get('info_json_path'), \
|
||||
'Archive readability info json should be in FileGroup.data'
|
||||
|
||||
# Upload an image file and verify it's added to the FileGroup.
|
||||
# Store file count before adding image
|
||||
file_count_before_image = len(archive.file_group.files)
|
||||
|
||||
# Create image using factory
|
||||
image_data = image_bytes_factory()
|
||||
|
||||
# Upload image with same stem as the Archive (using .jpg to avoid conflict with factory-created .png)
|
||||
image_filename = f'{singlefile_stem}.jpg'
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=image_filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(image_data))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=image_data, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Image upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Refresh session to get updated data
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
|
||||
# Assert screenshot properties exist
|
||||
assert archive.screenshot_path is not None, 'Archive.screenshot_path should exist'
|
||||
assert archive.screenshot_file is not None, 'Archive.screenshot_file should exist'
|
||||
assert archive.file_group.data.get('screenshot_path') == archive.screenshot_path, 'Archive screenshot should be in FileGroup.data'
|
||||
|
||||
# Assert image is in FileGroup.files
|
||||
file_paths = [f['path'] for f in archive.file_group.files]
|
||||
assert archive.screenshot_path in file_paths, 'Screenshot should be in FileGroup.files'
|
||||
|
||||
# Assert FileGroup has correct number of files
|
||||
assert len(archive.file_group.files) == file_count_before_image + 1, \
|
||||
f'FileGroup should have {file_count_before_image + 1} files after adding image'
|
||||
|
|
|
|||
|
|
@ -100,8 +100,12 @@ class Video(ModelHelper, Base):
|
|||
|
||||
@staticmethod
|
||||
def do_model(file_group: FileGroup, session: Session) -> 'Video':
|
||||
video = Video(file_group_id=file_group.id, file_group=file_group)
|
||||
session.add(video)
|
||||
# Check if a Video already exists for this FileGroup
|
||||
video = session.query(Video).filter_by(file_group_id=file_group.id).one_or_none()
|
||||
if not video:
|
||||
# Create new Video if it doesn't exist
|
||||
video = Video(file_group_id=file_group.id, file_group=file_group)
|
||||
session.add(video)
|
||||
video.validate()
|
||||
file_group.indexed = True
|
||||
video.flush()
|
||||
|
|
@ -239,6 +243,15 @@ class Video(ModelHelper, Base):
|
|||
self.channel = channel
|
||||
logger.debug(f'{self} has Channel {channel}')
|
||||
|
||||
# Set FileGroup.data with cached file paths for consistency with Archive
|
||||
self.file_group.data = {
|
||||
'id': self.id,
|
||||
'video_path': self.video_path,
|
||||
'info_json_path': self.info_json_path,
|
||||
'poster_path': self.poster_path,
|
||||
'caption_paths': self.caption_paths,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
|
|
|
|||
|
|
@ -361,3 +361,161 @@ async def test_video_file_format(async_client, test_session, fake_now):
|
|||
request, response = await async_client.post('/api/videos/file_format', json=body)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json['error'] == 'Filename must end with .%(ext)s'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_upload_file_tracking(test_session, async_client, video_factory, await_switches, make_multipart_form,
|
||||
image_bytes_factory):
|
||||
"""Test that uploading info.json, poster, and caption files via /api/files/upload properly tracks them in Video and FileGroup."""
|
||||
from wrolpi.common import get_relative_to_media_directory
|
||||
|
||||
# Step 1: Create a Video with just a video file (no info.json, poster, or captions)
|
||||
video = video_factory(with_video_file=True, title='file_tracking_test')
|
||||
test_session.commit()
|
||||
|
||||
# Step 2: Verify Video and FileGroup are created, but optional files do not exist
|
||||
file_group = video.file_group
|
||||
assert not video.info_json_path, 'info.json should not exist yet.'
|
||||
assert not video.poster_path, 'Poster should not exist yet.'
|
||||
assert not video.caption_paths, 'Captions should not exist yet.'
|
||||
assert file_group.files is not None, 'FileGroup should have files array'
|
||||
initial_file_count = len(file_group.files)
|
||||
|
||||
# Step 3: Upload info.json via /api/files/upload
|
||||
# Extract the video stem (e.g., "file_tracking_test")
|
||||
video_stem = video.video_path.stem
|
||||
|
||||
# Create info.json content
|
||||
info_json_data = {
|
||||
'title': 'file_tracking_test',
|
||||
'duration': 60,
|
||||
'uploader': 'test_uploader'
|
||||
}
|
||||
info_json_bytes = json.dumps(info_json_data).encode()
|
||||
|
||||
# Get relative destination directory
|
||||
destination = str(get_relative_to_media_directory(video.video_path.parent))
|
||||
filename = f'{video_stem}.info.json'
|
||||
|
||||
# POST to /api/files/upload
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(info_json_bytes))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=info_json_bytes, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Step 4: Verify info.json is tracked
|
||||
# Refresh the session to get updated data
|
||||
test_session.expire_all()
|
||||
video = test_session.query(Video).filter_by(id=video.id).one()
|
||||
|
||||
# Assert info.json properties exist
|
||||
assert video.info_json_path is not None, 'Video.info_json_path should exist'
|
||||
assert video.info_json_file is not None, 'Video.info_json_file should exist'
|
||||
|
||||
# Assert FileGroup has the correct number of files
|
||||
assert len(video.file_group.files) == initial_file_count + 1, \
|
||||
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
|
||||
assert video.file_group.data.get('info_json_path'), \
|
||||
'Video info.json should be in FileGroup.data'
|
||||
|
||||
# Step 5: Upload poster image and verify it's added to the FileGroup
|
||||
file_count_before_poster = len(video.file_group.files)
|
||||
|
||||
# Create image using factory
|
||||
image_data = image_bytes_factory()
|
||||
|
||||
# Upload image with same stem as the Video (using .jpg)
|
||||
poster_filename = f'{video_stem}.jpg'
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=poster_filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(image_data))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=image_data, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Poster upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Refresh session to get updated data
|
||||
test_session.expire_all()
|
||||
video = test_session.query(Video).filter_by(id=video.id).one()
|
||||
|
||||
# Assert poster properties exist
|
||||
assert video.poster_path is not None, 'Video.poster_path should exist'
|
||||
assert video.poster_file is not None, 'Video.poster_file should exist'
|
||||
assert video.file_group.data.get('poster_path') == video.poster_path, 'Video poster should be in FileGroup.data'
|
||||
|
||||
# Assert poster is in FileGroup.files
|
||||
file_paths = [f['path'] for f in video.file_group.files]
|
||||
assert video.poster_path in file_paths, 'Poster should be in FileGroup.files'
|
||||
|
||||
# Assert FileGroup has correct number of files
|
||||
assert len(video.file_group.files) == file_count_before_poster + 1, \
|
||||
f'FileGroup should have {file_count_before_poster + 1} files after adding poster'
|
||||
|
||||
# Step 6: Upload caption file and verify it's added to the FileGroup
|
||||
file_count_before_caption = len(video.file_group.files)
|
||||
|
||||
# Create caption content
|
||||
caption_content = """WEBVTT
|
||||
|
||||
00:00:00.000 --> 00:00:02.000
|
||||
This is a test caption.
|
||||
|
||||
00:00:02.000 --> 00:00:04.000
|
||||
Another line of captions.
|
||||
"""
|
||||
|
||||
# Upload caption with same stem as the Video
|
||||
caption_filename = f'{video_stem}.en.vtt'
|
||||
caption_bytes = caption_content.encode()
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=caption_filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(caption_bytes))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=caption_bytes, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Caption upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Refresh session to get updated data
|
||||
test_session.expire_all()
|
||||
video = test_session.query(Video).filter_by(id=video.id).one()
|
||||
|
||||
# Assert caption properties exist
|
||||
assert video.caption_paths, 'Video.caption_paths should exist'
|
||||
assert len(video.caption_paths) == 1, 'Should have exactly one caption file'
|
||||
assert video.caption_files, 'Video.caption_files should exist'
|
||||
assert video.file_group.data.get('caption_paths'), 'Video captions should be in FileGroup.data'
|
||||
|
||||
# Assert caption is in FileGroup.files
|
||||
file_paths = [f['path'] for f in video.file_group.files]
|
||||
assert video.caption_paths[0] in file_paths, 'Caption should be in FileGroup.files'
|
||||
|
||||
# Assert FileGroup has correct number of files
|
||||
assert len(video.file_group.files) == file_count_before_caption + 1, \
|
||||
f'FileGroup should have {file_count_before_caption + 1} files after adding caption'
|
||||
|
|
|
|||
|
|
@ -317,7 +317,7 @@ def test_browser_profile_to_yt_dlp_arg(file_name, expected):
|
|||
assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected
|
||||
|
||||
|
||||
def test_video_location(test_session, video_factory, channel_factory):
|
||||
def test_video_location(test_session, video_factory, channel_factory, async_client):
|
||||
# Video location without a channel, it opens the preview.
|
||||
video1 = video_factory(title='vid1')
|
||||
assert video1.location == '/videos/video/1'
|
||||
|
|
|
|||
|
|
@ -43,8 +43,12 @@ __all__ = [
|
|||
|
||||
def model_zim(file_group: FileGroup, session: Session) -> Zim:
|
||||
"""Create a Zim record for the provided FileGroup."""
|
||||
zim = Zim(file_group=file_group, path=file_group.primary_path)
|
||||
session.add(zim)
|
||||
# Check if a Zim already exists for this FileGroup
|
||||
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
|
||||
if not zim:
|
||||
# Create new Zim if it doesn't exist
|
||||
zim = Zim(file_group=file_group, path=file_group.primary_path)
|
||||
session.add(zim)
|
||||
file_group.title = file_group.primary_path.name
|
||||
file_group.a_text = split_file_name_words(file_group.primary_path.name)
|
||||
file_group.indexed = True
|
||||
|
|
|
|||
|
|
@ -680,8 +680,8 @@ def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.P
|
|||
|
||||
@pytest.fixture
|
||||
def image_bytes_factory():
|
||||
def _() -> bytes:
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
|
||||
def _(suffix: str = '.png') -> bytes:
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix) as fh:
|
||||
Image.new('RGB', (1, 1), color='grey').save(fh)
|
||||
fh.seek(0)
|
||||
image = fh.read()
|
||||
|
|
|
|||
|
|
@ -228,8 +228,12 @@ class EBook(ModelHelper, Base):
|
|||
|
||||
def model_ebook(file_group: FileGroup, session: Session) -> EBook:
|
||||
"""Creates an EBook model based off a File. Searches for its cover in the provided `files`."""
|
||||
ebook = EBook(file_group=file_group)
|
||||
session.add(ebook)
|
||||
# Check if an EBook already exists for this FileGroup
|
||||
ebook = session.query(EBook).filter_by(file_group_id=file_group.id).one_or_none()
|
||||
if not ebook:
|
||||
# Create new EBook if it doesn't exist
|
||||
ebook = EBook(file_group=file_group)
|
||||
session.add(ebook)
|
||||
|
||||
# Multiple formats may be available.
|
||||
epub_file = next(iter(ebook.file_group.my_epub_files()), None)
|
||||
|
|
|
|||
|
|
@ -1582,61 +1582,61 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
|
|||
logger.warning('upsert_file called, but all files are in ignored directories!')
|
||||
raise IgnoredDirectoryError(f'all files are in ignored directories: {file}')
|
||||
|
||||
for i in range(2):
|
||||
# Try multiple times because uploads happen concurrently and may conflict.
|
||||
# TODO convert uploads to synchronous in UI.
|
||||
with get_db_session() as session:
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
# Re-index the contents of the file.
|
||||
# Use a single session context for all database operations
|
||||
with get_db_session(commit=True) as session:
|
||||
# Create/update FileGroup with retry logic
|
||||
file_group = None
|
||||
for i in range(2):
|
||||
try:
|
||||
session.flush([file_group, ])
|
||||
file_group_id = file_group.id
|
||||
session.commit()
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
session.flush([file_group])
|
||||
break
|
||||
except IntegrityError:
|
||||
# Another process inserted this FileGroup.
|
||||
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
|
||||
session.rollback()
|
||||
if i == 1:
|
||||
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
|
||||
continue
|
||||
else:
|
||||
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
|
||||
|
||||
logger.debug(f'upsert_file: {file_group}')
|
||||
try:
|
||||
with get_db_session(commit=True) as session:
|
||||
file_group = FileGroup.find_by_id(file_group_id, session)
|
||||
if not file_group:
|
||||
raise RuntimeError(f'upsert_file failed to create FileGroup! {file}')
|
||||
|
||||
logger.debug(f'upsert_file: {file_group}')
|
||||
|
||||
# Model the FileGroup (creates Archive/Video/etc. and sets indexed=True)
|
||||
try:
|
||||
file_group.do_model(session)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
# If user uploads a file, then remove it from the download skip list so comments can be downloaded. Modify the
|
||||
# download (if any) so that the user can click on it and view the uploaded file.
|
||||
if url := file_group.url if file_group and file_group.url else None:
|
||||
if download_manager.is_skipped(url):
|
||||
download_manager.remove_from_skip_list(url)
|
||||
with get_db_session() as session:
|
||||
if download := Download.get_by_url(url, session):
|
||||
# If user uploads a file, then remove it from the download skip list so comments can be downloaded.
|
||||
# Modify the download (if any) so that the user can click on it and view the uploaded file.
|
||||
if file_group.url:
|
||||
if download_manager.is_skipped(file_group.url):
|
||||
download_manager.remove_from_skip_list(file_group.url)
|
||||
if download := Download.get_by_url(file_group.url, session):
|
||||
# Mark download as completed
|
||||
download.complete()
|
||||
# Link the download to the location of the uploaded file.
|
||||
model = file_group.get_model_record()
|
||||
download.location = model.location if model else file_group.location
|
||||
session.commit()
|
||||
|
||||
upsert_directories([], file.parents)
|
||||
|
||||
session.commit()
|
||||
|
||||
if tag_names:
|
||||
with get_db_session(commit=True) as session:
|
||||
file_group = FileGroup.find_by_id(file_group_id, session)
|
||||
# Add tags if provided
|
||||
if tag_names:
|
||||
for tag_name in tag_names:
|
||||
if tag_name not in file_group.tag_names:
|
||||
tag = Tag.get_by_name(tag_name)
|
||||
file_group.add_tag(tag.id)
|
||||
tag = Tag.get_by_name(tag_name, session)
|
||||
file_group.add_tag(tag.id, session=session)
|
||||
|
||||
# Commit all changes in one transaction
|
||||
session.commit()
|
||||
|
||||
# Upsert directories after FileGroup is committed
|
||||
upsert_directories([], file.parents)
|
||||
|
||||
file_group.flush()
|
||||
return file_group
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -513,13 +513,10 @@ class FileGroup(ModelHelper, Base):
|
|||
def do_model(self, session: Session) -> Optional[ModelHelper]:
|
||||
"""Get/Create the Model record (Video/Archive/etc.) for this FileGroup, if any."""
|
||||
if model_class := self.get_model_class():
|
||||
if model := model_class.get_by_path(self.primary_path, session):
|
||||
# Already modeled.
|
||||
logger.debug(f'already modeled: {self} {model}')
|
||||
return model
|
||||
# Create new model (Video/Archive/Ebook).
|
||||
# Always call do_model() to ensure FileGroup.data is updated when files change.
|
||||
# The model's do_model() should be idempotent (safe to call multiple times).
|
||||
model = model_class.do_model(self, session)
|
||||
logger.debug(f'new model: {self} {model}')
|
||||
logger.debug(f'modeled: {self} {model}')
|
||||
return model
|
||||
|
||||
# Index the file if no models are available.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue