Fixing bug where Archive screenshots would not be tracked when a user uploads a screenshot.
Some checks failed
CodeQL / Analyze (push) Has been cancelled

This commit is contained in:
Roland 2025-10-12 17:19:54 -06:00
parent ecac129816
commit c635267eda
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
13 changed files with 368 additions and 60 deletions

View file

@ -181,8 +181,12 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
if readability_txt_path:
contents = get_article(readability_txt_path)
archive = Archive(file_group_id=file_group_id, file_group=file_group)
session.add(archive)
# Check if an Archive already exists for this FileGroup
archive = session.query(Archive).filter_by(file_group_id=file_group_id).one_or_none()
if not archive:
# Create new Archive if it doesn't exist
archive = Archive(file_group_id=file_group_id, file_group=file_group)
session.add(archive)
archive.validate()
archive.flush()
@ -196,6 +200,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
'readability_txt_path': archive.readability_txt_path,
'screenshot_path': archive.screenshot_path,
'singlefile_path': archive.singlefile_path,
'info_json_path': archive.info_json_path,
}
return archive

View file

@ -104,7 +104,11 @@ async def singlefile_upload_switch_handler(url=None):
logger.trace(f'singlefile_upload_switch_handler called on empty queue')
return
q_size = q.qsize()
try:
q_size = q.qsize()
except NotImplementedError:
# qsize() is not implemented on macOS
q_size = '?'
logger.info(f'singlefile_upload_switch_handler queue size: {q_size}')
try:

View file

@ -19,7 +19,7 @@ def archive_directory(test_directory) -> pathlib.Path:
@pytest.fixture
def archive_factory(test_session, archive_directory, make_files_structure):
def archive_factory(test_session, archive_directory, make_files_structure, image_bytes_factory):
def time_generator():
timestamp = datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC)
while True:
@ -29,7 +29,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
now = time_generator()
def _(domain: str = None, url: str = None, title: str = 'NA', contents: str = None,
singlefile_contents: str = None, tag_names: List[str] = None) -> Archive:
singlefile_contents: str = None, tag_names: List[str] = None, screenshot: bool = True) -> Archive:
if domain:
assert '/' not in domain
@ -49,10 +49,9 @@ def archive_factory(test_session, archive_directory, make_files_structure):
if url:
json_contents['url'] = url
json_contents = json.dumps(json_contents)
singlefile_path, screenshot_path, readability_path, readability_txt_path, readability_json_path = \
singlefile_path, readability_path, readability_txt_path, readability_json_path = \
make_files_structure({
str(domain_dir / f'{timestamp}_{title}.html'): singlefile_contents or '<html></html>',
str(domain_dir / f'{timestamp}_{title}.png'): None,
str(domain_dir / f'{timestamp}_{title}.readability.html'): '<html></html>',
str(domain_dir / f'{timestamp}_{title}.readability.txt'): contents,
str(domain_dir / f'{timestamp}_{title}.readability.json'): json_contents,
@ -61,6 +60,11 @@ def archive_factory(test_session, archive_directory, make_files_structure):
if domain:
domain = test_session.query(Domain).filter_by(domain=domain).one_or_none()
screenshot_path = None
if screenshot:
screenshot_path = domain_dir / f'{timestamp}_{title}.png'
screenshot_path.write_bytes(image_bytes_factory())
if not domain and domain_dir.name != 'NA':
domain = Domain(
domain=domain_dir.name,

View file

@ -197,6 +197,18 @@ class Archive(Base, ModelHelper):
if screenshot_file := self.screenshot_file:
return screenshot_file['path']
@property
def info_json_file(self) -> Optional[dict]:
files = self.file_group.my_files()
for file in files:
if file['path'].name.endswith('.info.json'):
return file
@property
def info_json_path(self) -> Optional[pathlib.Path]:
if info_json_file := self.info_json_file:
return info_json_file['path']
def apply_readability_data(self):
"""Read the Readability JSON file, apply its contents to this record."""
readability_json_path = self.readability_json_path

View file

@ -4,8 +4,11 @@ from http import HTTPStatus
import pytest
from modules.archive import lib, Archive
from wrolpi.files.models import FileGroup
from wrolpi.test.common import skip_circleci, skip_macos
from wrolpi.common import get_relative_to_media_directory
def check_results(test_client, data, ids):
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
@ -202,7 +205,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
request, response = test_client.get(f'/api/archive/domains')
assert response.status_code == HTTPStatus.OK
assert response.json['domains'][0]['domain'] == 'example.com'
assert response.json['domains'][0]['size'] == 116
assert response.json['domains'][0]['size'] == 254
# Deleting works.
request, response = test_client.delete(f'/api/archive/{archive1.id}')
@ -257,3 +260,107 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
events = list(events_history)
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'
@pytest.mark.asyncio
async def test_archive_upload_file_tracking(test_session, async_client, archive_directory, archive_factory,
await_switches, make_multipart_form, image_bytes_factory):
"""Test that uploading an info.json and image file via /api/files/upload properly tracks them in Archive and FileGroup."""
archive = archive_factory('example.com', 'https://example.com/test-url', 'tracking test', screenshot=False)
test_session.commit()
# Verify Archive and FileGroup are created, but screenshot does not exist.
file_group = archive.file_group
assert not archive.screenshot_path, 'Screenshot should not exist yet.'
assert file_group.files is not None, 'FileGroup should have files array'
initial_file_count = len(file_group.files)
# Upload info.json via /api/files/upload
# Extract the prefix from singlefile path (e.g., "2000-01-01-00-00-01_tracking test")
singlefile_stem = archive.singlefile_path.stem # e.g., "2000-01-01-00-00-01_tracking test"
# Create info.json content
info_json_data = {
'title': 'test',
'url': 'https://example.com/test-url',
'custom_field': 'value'
}
info_json_bytes = json.dumps(info_json_data).encode()
# Get relative destination directory
destination = str(get_relative_to_media_directory(archive.singlefile_path.parent))
filename = f'{singlefile_stem}.info.json'
# POST to /api/files/upload
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(info_json_bytes))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=info_json_bytes, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Step 4: Verify info.json is tracked
# Refresh the session to get updated data
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Assert info.json properties exist
assert archive.info_json_path is not None, 'Archive.info_json_path should exist'
assert archive.info_json_file is not None, 'Archive.info_json_file should exist'
# Assert FileGroup has the correct number of files.
assert len(archive.file_group.files) == initial_file_count + 1, \
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
assert archive.file_group.data.get('info_json_path'), \
'Archive readability info json should be in FileGroup.data'
# Upload an image file and verify it's added to the FileGroup.
# Store file count before adding image
file_count_before_image = len(archive.file_group.files)
# Create image using factory
image_data = image_bytes_factory()
# Upload image with same stem as the Archive (using .jpg to avoid conflict with factory-created .png)
image_filename = f'{singlefile_stem}.jpg'
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=image_filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(image_data))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=image_data, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Image upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Refresh session to get updated data
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Assert screenshot properties exist
assert archive.screenshot_path is not None, 'Archive.screenshot_path should exist'
assert archive.screenshot_file is not None, 'Archive.screenshot_file should exist'
assert archive.file_group.data.get('screenshot_path') == archive.screenshot_path, 'Archive screenshot should be in FileGroup.data'
# Assert image is in FileGroup.files
file_paths = [f['path'] for f in archive.file_group.files]
assert archive.screenshot_path in file_paths, 'Screenshot should be in FileGroup.files'
# Assert FileGroup has correct number of files
assert len(archive.file_group.files) == file_count_before_image + 1, \
f'FileGroup should have {file_count_before_image + 1} files after adding image'

View file

@ -100,8 +100,12 @@ class Video(ModelHelper, Base):
@staticmethod
def do_model(file_group: FileGroup, session: Session) -> 'Video':
video = Video(file_group_id=file_group.id, file_group=file_group)
session.add(video)
# Check if a Video already exists for this FileGroup
video = session.query(Video).filter_by(file_group_id=file_group.id).one_or_none()
if not video:
# Create new Video if it doesn't exist
video = Video(file_group_id=file_group.id, file_group=file_group)
session.add(video)
video.validate()
file_group.indexed = True
video.flush()
@ -239,6 +243,15 @@ class Video(ModelHelper, Base):
self.channel = channel
logger.debug(f'{self} has Channel {channel}')
# Set FileGroup.data with cached file paths for consistency with Archive
self.file_group.data = {
'id': self.id,
'video_path': self.video_path,
'info_json_path': self.info_json_path,
'poster_path': self.poster_path,
'caption_paths': self.caption_paths,
}
@staticmethod
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
file_group = FileGroup.from_paths(session, *paths)

View file

@ -361,3 +361,161 @@ async def test_video_file_format(async_client, test_session, fake_now):
request, response = await async_client.post('/api/videos/file_format', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json['error'] == 'Filename must end with .%(ext)s'
@pytest.mark.asyncio
async def test_video_upload_file_tracking(test_session, async_client, video_factory, await_switches, make_multipart_form,
image_bytes_factory):
"""Test that uploading info.json, poster, and caption files via /api/files/upload properly tracks them in Video and FileGroup."""
from wrolpi.common import get_relative_to_media_directory
# Step 1: Create a Video with just a video file (no info.json, poster, or captions)
video = video_factory(with_video_file=True, title='file_tracking_test')
test_session.commit()
# Step 2: Verify Video and FileGroup are created, but optional files do not exist
file_group = video.file_group
assert not video.info_json_path, 'info.json should not exist yet.'
assert not video.poster_path, 'Poster should not exist yet.'
assert not video.caption_paths, 'Captions should not exist yet.'
assert file_group.files is not None, 'FileGroup should have files array'
initial_file_count = len(file_group.files)
# Step 3: Upload info.json via /api/files/upload
# Extract the video stem (e.g., "file_tracking_test")
video_stem = video.video_path.stem
# Create info.json content
info_json_data = {
'title': 'file_tracking_test',
'duration': 60,
'uploader': 'test_uploader'
}
info_json_bytes = json.dumps(info_json_data).encode()
# Get relative destination directory
destination = str(get_relative_to_media_directory(video.video_path.parent))
filename = f'{video_stem}.info.json'
# POST to /api/files/upload
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(info_json_bytes))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=info_json_bytes, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Step 4: Verify info.json is tracked
# Refresh the session to get updated data
test_session.expire_all()
video = test_session.query(Video).filter_by(id=video.id).one()
# Assert info.json properties exist
assert video.info_json_path is not None, 'Video.info_json_path should exist'
assert video.info_json_file is not None, 'Video.info_json_file should exist'
# Assert FileGroup has the correct number of files
assert len(video.file_group.files) == initial_file_count + 1, \
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
assert video.file_group.data.get('info_json_path'), \
'Video info.json should be in FileGroup.data'
# Step 5: Upload poster image and verify it's added to the FileGroup
file_count_before_poster = len(video.file_group.files)
# Create image using factory
image_data = image_bytes_factory()
# Upload image with same stem as the Video (using .jpg)
poster_filename = f'{video_stem}.jpg'
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=poster_filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(image_data))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=image_data, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Poster upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Refresh session to get updated data
test_session.expire_all()
video = test_session.query(Video).filter_by(id=video.id).one()
# Assert poster properties exist
assert video.poster_path is not None, 'Video.poster_path should exist'
assert video.poster_file is not None, 'Video.poster_file should exist'
assert video.file_group.data.get('poster_path') == video.poster_path, 'Video poster should be in FileGroup.data'
# Assert poster is in FileGroup.files
file_paths = [f['path'] for f in video.file_group.files]
assert video.poster_path in file_paths, 'Poster should be in FileGroup.files'
# Assert FileGroup has correct number of files
assert len(video.file_group.files) == file_count_before_poster + 1, \
f'FileGroup should have {file_count_before_poster + 1} files after adding poster'
# Step 6: Upload caption file and verify it's added to the FileGroup
file_count_before_caption = len(video.file_group.files)
# Create caption content
caption_content = """WEBVTT
00:00:00.000 --> 00:00:02.000
This is a test caption.
00:00:02.000 --> 00:00:04.000
Another line of captions.
"""
# Upload caption with same stem as the Video
caption_filename = f'{video_stem}.en.vtt'
caption_bytes = caption_content.encode()
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=caption_filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(caption_bytes))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=caption_bytes, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Caption upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Refresh session to get updated data
test_session.expire_all()
video = test_session.query(Video).filter_by(id=video.id).one()
# Assert caption properties exist
assert video.caption_paths, 'Video.caption_paths should exist'
assert len(video.caption_paths) == 1, 'Should have exactly one caption file'
assert video.caption_files, 'Video.caption_files should exist'
assert video.file_group.data.get('caption_paths'), 'Video captions should be in FileGroup.data'
# Assert caption is in FileGroup.files
file_paths = [f['path'] for f in video.file_group.files]
assert video.caption_paths[0] in file_paths, 'Caption should be in FileGroup.files'
# Assert FileGroup has correct number of files
assert len(video.file_group.files) == file_count_before_caption + 1, \
f'FileGroup should have {file_count_before_caption + 1} files after adding caption'

View file

@ -317,7 +317,7 @@ def test_browser_profile_to_yt_dlp_arg(file_name, expected):
assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected
def test_video_location(test_session, video_factory, channel_factory):
def test_video_location(test_session, video_factory, channel_factory, async_client):
# Video location without a channel, it opens the preview.
video1 = video_factory(title='vid1')
assert video1.location == '/videos/video/1'

View file

@ -43,8 +43,12 @@ __all__ = [
def model_zim(file_group: FileGroup, session: Session) -> Zim:
"""Create a Zim record for the provided FileGroup."""
zim = Zim(file_group=file_group, path=file_group.primary_path)
session.add(zim)
# Check if a Zim already exists for this FileGroup
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
if not zim:
# Create new Zim if it doesn't exist
zim = Zim(file_group=file_group, path=file_group.primary_path)
session.add(zim)
file_group.title = file_group.primary_path.name
file_group.a_text = split_file_name_words(file_group.primary_path.name)
file_group.indexed = True

View file

@ -680,8 +680,8 @@ def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.P
@pytest.fixture
def image_bytes_factory():
def _() -> bytes:
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
def _(suffix: str = '.png') -> bytes:
with tempfile.NamedTemporaryFile(suffix=suffix) as fh:
Image.new('RGB', (1, 1), color='grey').save(fh)
fh.seek(0)
image = fh.read()

View file

@ -228,8 +228,12 @@ class EBook(ModelHelper, Base):
def model_ebook(file_group: FileGroup, session: Session) -> EBook:
"""Creates an EBook model based off a File. Searches for its cover in the provided `files`."""
ebook = EBook(file_group=file_group)
session.add(ebook)
# Check if an EBook already exists for this FileGroup
ebook = session.query(EBook).filter_by(file_group_id=file_group.id).one_or_none()
if not ebook:
# Create new EBook if it doesn't exist
ebook = EBook(file_group=file_group)
session.add(ebook)
# Multiple formats may be available.
epub_file = next(iter(ebook.file_group.my_epub_files()), None)

View file

@ -1582,61 +1582,61 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
logger.warning('upsert_file called, but all files are in ignored directories!')
raise IgnoredDirectoryError(f'all files are in ignored directories: {file}')
for i in range(2):
# Try multiple times because uploads happen concurrently and may conflict.
# TODO convert uploads to synchronous in UI.
with get_db_session() as session:
file_group = FileGroup.from_paths(session, *paths)
# Re-index the contents of the file.
# Use a single session context for all database operations
with get_db_session(commit=True) as session:
# Create/update FileGroup with retry logic
file_group = None
for i in range(2):
try:
session.flush([file_group, ])
file_group_id = file_group.id
session.commit()
file_group = FileGroup.from_paths(session, *paths)
session.flush([file_group])
break
except IntegrityError:
# Another process inserted this FileGroup.
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
session.rollback()
if i == 1:
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
continue
else:
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
logger.debug(f'upsert_file: {file_group}')
try:
with get_db_session(commit=True) as session:
file_group = FileGroup.find_by_id(file_group_id, session)
if not file_group:
raise RuntimeError(f'upsert_file failed to create FileGroup! {file}')
logger.debug(f'upsert_file: {file_group}')
# Model the FileGroup (creates Archive/Video/etc. and sets indexed=True)
try:
file_group.do_model(session)
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
# If user uploads a file, then remove it from the download skip list so comments can be downloaded. Modify the
# download (if any) so that the user can click on it and view the uploaded file.
if url := file_group.url if file_group and file_group.url else None:
if download_manager.is_skipped(url):
download_manager.remove_from_skip_list(url)
with get_db_session() as session:
if download := Download.get_by_url(url, session):
# If user uploads a file, then remove it from the download skip list so comments can be downloaded.
# Modify the download (if any) so that the user can click on it and view the uploaded file.
if file_group.url:
if download_manager.is_skipped(file_group.url):
download_manager.remove_from_skip_list(file_group.url)
if download := Download.get_by_url(file_group.url, session):
# Mark download as completed
download.complete()
# Link the download to the location of the uploaded file.
model = file_group.get_model_record()
download.location = model.location if model else file_group.location
session.commit()
upsert_directories([], file.parents)
session.commit()
if tag_names:
with get_db_session(commit=True) as session:
file_group = FileGroup.find_by_id(file_group_id, session)
# Add tags if provided
if tag_names:
for tag_name in tag_names:
if tag_name not in file_group.tag_names:
tag = Tag.get_by_name(tag_name)
file_group.add_tag(tag.id)
tag = Tag.get_by_name(tag_name, session)
file_group.add_tag(tag.id, session=session)
# Commit all changes in one transaction
session.commit()
# Upsert directories after FileGroup is committed
upsert_directories([], file.parents)
file_group.flush()
return file_group

View file

@ -513,13 +513,10 @@ class FileGroup(ModelHelper, Base):
def do_model(self, session: Session) -> Optional[ModelHelper]:
"""Get/Create the Model record (Video/Archive/etc.) for this FileGroup, if any."""
if model_class := self.get_model_class():
if model := model_class.get_by_path(self.primary_path, session):
# Already modeled.
logger.debug(f'already modeled: {self} {model}')
return model
# Create new model (Video/Archive/Ebook).
# Always call do_model() to ensure FileGroup.data is updated when files change.
# The model's do_model() should be idempotent (safe to call multiple times).
model = model_class.do_model(self, session)
logger.debug(f'new model: {self} {model}')
logger.debug(f'modeled: {self} {model}')
return model
# Index the file if no models are available.