From c635267edac471404524a171367928bf1fa2b183 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 12 Oct 2025 17:19:54 -0600 Subject: [PATCH] Fixing bug where Archive screenshots would not be tracked when a user uploads a screenshot. --- modules/archive/__init__.py | 9 +- modules/archive/api.py | 6 +- modules/archive/conftest.py | 12 ++- modules/archive/models.py | 12 +++ modules/archive/test/test_api.py | 109 ++++++++++++++++++++- modules/videos/models.py | 17 +++- modules/videos/test/test_api.py | 158 +++++++++++++++++++++++++++++++ modules/videos/test/test_lib.py | 2 +- modules/zim/lib.py | 8 +- wrolpi/conftest.py | 4 +- wrolpi/files/ebooks.py | 8 +- wrolpi/files/lib.py | 74 +++++++-------- wrolpi/files/models.py | 9 +- 13 files changed, 368 insertions(+), 60 deletions(-) diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 2ced4067..05426e00 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -181,8 +181,12 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive: if readability_txt_path: contents = get_article(readability_txt_path) - archive = Archive(file_group_id=file_group_id, file_group=file_group) - session.add(archive) + # Check if an Archive already exists for this FileGroup + archive = session.query(Archive).filter_by(file_group_id=file_group_id).one_or_none() + if not archive: + # Create new Archive if it doesn't exist + archive = Archive(file_group_id=file_group_id, file_group=file_group) + session.add(archive) archive.validate() archive.flush() @@ -196,6 +200,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive: 'readability_txt_path': archive.readability_txt_path, 'screenshot_path': archive.screenshot_path, 'singlefile_path': archive.singlefile_path, + 'info_json_path': archive.info_json_path, } return archive diff --git a/modules/archive/api.py b/modules/archive/api.py index a649bd6e..8ea6469b 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -104,7 +104,11 @@ async def singlefile_upload_switch_handler(url=None): logger.trace(f'singlefile_upload_switch_handler called on empty queue') return - q_size = q.qsize() + try: + q_size = q.qsize() + except NotImplementedError: + # qsize() is not implemented on macOS + q_size = '?' logger.info(f'singlefile_upload_switch_handler queue size: {q_size}') try: diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py index 0898d8ee..15667dee 100644 --- a/modules/archive/conftest.py +++ b/modules/archive/conftest.py @@ -19,7 +19,7 @@ def archive_directory(test_directory) -> pathlib.Path: @pytest.fixture -def archive_factory(test_session, archive_directory, make_files_structure): +def archive_factory(test_session, archive_directory, make_files_structure, image_bytes_factory): def time_generator(): timestamp = datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC) while True: @@ -29,7 +29,7 @@ def archive_factory(test_session, archive_directory, make_files_structure): now = time_generator() def _(domain: str = None, url: str = None, title: str = 'NA', contents: str = None, - singlefile_contents: str = None, tag_names: List[str] = None) -> Archive: + singlefile_contents: str = None, tag_names: List[str] = None, screenshot: bool = True) -> Archive: if domain: assert '/' not in domain @@ -49,10 +49,9 @@ def archive_factory(test_session, archive_directory, make_files_structure): if url: json_contents['url'] = url json_contents = json.dumps(json_contents) - singlefile_path, screenshot_path, readability_path, readability_txt_path, readability_json_path = \ + singlefile_path, readability_path, readability_txt_path, readability_json_path = \ make_files_structure({ str(domain_dir / f'{timestamp}_{title}.html'): singlefile_contents or '', - str(domain_dir / f'{timestamp}_{title}.png'): None, str(domain_dir / f'{timestamp}_{title}.readability.html'): '', str(domain_dir / f'{timestamp}_{title}.readability.txt'): contents, str(domain_dir / f'{timestamp}_{title}.readability.json'): json_contents, @@ -61,6 +60,11 @@ def archive_factory(test_session, archive_directory, make_files_structure): if domain: domain = test_session.query(Domain).filter_by(domain=domain).one_or_none() + screenshot_path = None + if screenshot: + screenshot_path = domain_dir / f'{timestamp}_{title}.png' + screenshot_path.write_bytes(image_bytes_factory()) + if not domain and domain_dir.name != 'NA': domain = Domain( domain=domain_dir.name, diff --git a/modules/archive/models.py b/modules/archive/models.py index 22f7f969..4f8fe647 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -197,6 +197,18 @@ class Archive(Base, ModelHelper): if screenshot_file := self.screenshot_file: return screenshot_file['path'] + @property + def info_json_file(self) -> Optional[dict]: + files = self.file_group.my_files() + for file in files: + if file['path'].name.endswith('.info.json'): + return file + + @property + def info_json_path(self) -> Optional[pathlib.Path]: + if info_json_file := self.info_json_file: + return info_json_file['path'] + def apply_readability_data(self): """Read the Readability JSON file, apply its contents to this record.""" readability_json_path = self.readability_json_path diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index 2dac5f84..3fdb0541 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -4,8 +4,11 @@ from http import HTTPStatus import pytest from modules.archive import lib, Archive +from wrolpi.files.models import FileGroup from wrolpi.test.common import skip_circleci, skip_macos +from wrolpi.common import get_relative_to_media_directory + def check_results(test_client, data, ids): request, response = test_client.post('/api/archive/search', content=json.dumps(data)) @@ -202,7 +205,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory): request, response = test_client.get(f'/api/archive/domains') assert response.status_code == HTTPStatus.OK assert response.json['domains'][0]['domain'] == 'example.com' - assert response.json['domains'][0]['size'] == 116 + assert response.json['domains'][0]['size'] == 254 # Deleting works. request, response = test_client.delete(f'/api/archive/{archive1.id}') @@ -257,3 +260,107 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted' events = list(events_history) assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.' + + +@pytest.mark.asyncio +async def test_archive_upload_file_tracking(test_session, async_client, archive_directory, archive_factory, + await_switches, make_multipart_form, image_bytes_factory): + """Test that uploading an info.json and image file via /api/files/upload properly tracks them in Archive and FileGroup.""" + archive = archive_factory('example.com', 'https://example.com/test-url', 'tracking test', screenshot=False) + test_session.commit() + + # Verify Archive and FileGroup are created, but screenshot does not exist. + file_group = archive.file_group + assert not archive.screenshot_path, 'Screenshot should not exist yet.' + assert file_group.files is not None, 'FileGroup should have files array' + initial_file_count = len(file_group.files) + + # Upload info.json via /api/files/upload + # Extract the prefix from singlefile path (e.g., "2000-01-01-00-00-01_tracking test") + singlefile_stem = archive.singlefile_path.stem # e.g., "2000-01-01-00-00-01_tracking test" + + # Create info.json content + info_json_data = { + 'title': 'test', + 'url': 'https://example.com/test-url', + 'custom_field': 'value' + } + info_json_bytes = json.dumps(info_json_data).encode() + + # Get relative destination directory + destination = str(get_relative_to_media_directory(archive.singlefile_path.parent)) + filename = f'{singlefile_stem}.info.json' + + # POST to /api/files/upload + forms = [ + dict(name='destination', value=destination), + dict(name='filename', value=filename), + dict(name='chunkNumber', value='0'), + dict(name='totalChunks', value='0'), + dict(name='chunkSize', value=str(len(info_json_bytes))), + dict(name='overwrite', value='false'), + dict(name='chunk', value=info_json_bytes, filename='chunk'), + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}' + + await await_switches() + + # Step 4: Verify info.json is tracked + # Refresh the session to get updated data + test_session.expire_all() + archive = test_session.query(Archive).filter_by(id=archive.id).one() + + # Assert info.json properties exist + assert archive.info_json_path is not None, 'Archive.info_json_path should exist' + assert archive.info_json_file is not None, 'Archive.info_json_file should exist' + + # Assert FileGroup has the correct number of files. + assert len(archive.file_group.files) == initial_file_count + 1, \ + f'FileGroup should have {initial_file_count + 1} files after adding info.json' + assert archive.file_group.data.get('info_json_path'), \ + 'Archive readability info json should be in FileGroup.data' + + # Upload an image file and verify it's added to the FileGroup. + # Store file count before adding image + file_count_before_image = len(archive.file_group.files) + + # Create image using factory + image_data = image_bytes_factory() + + # Upload image with same stem as the Archive (using .jpg to avoid conflict with factory-created .png) + image_filename = f'{singlefile_stem}.jpg' + forms = [ + dict(name='destination', value=destination), + dict(name='filename', value=image_filename), + dict(name='chunkNumber', value='0'), + dict(name='totalChunks', value='0'), + dict(name='chunkSize', value=str(len(image_data))), + dict(name='overwrite', value='false'), + dict(name='chunk', value=image_data, filename='chunk'), + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.CREATED, f'Image upload failed with status {response.status_code}: {response.json}' + + await await_switches() + + # Refresh session to get updated data + test_session.expire_all() + archive = test_session.query(Archive).filter_by(id=archive.id).one() + + # Assert screenshot properties exist + assert archive.screenshot_path is not None, 'Archive.screenshot_path should exist' + assert archive.screenshot_file is not None, 'Archive.screenshot_file should exist' + assert archive.file_group.data.get('screenshot_path') == archive.screenshot_path, 'Archive screenshot should be in FileGroup.data' + + # Assert image is in FileGroup.files + file_paths = [f['path'] for f in archive.file_group.files] + assert archive.screenshot_path in file_paths, 'Screenshot should be in FileGroup.files' + + # Assert FileGroup has correct number of files + assert len(archive.file_group.files) == file_count_before_image + 1, \ + f'FileGroup should have {file_count_before_image + 1} files after adding image' diff --git a/modules/videos/models.py b/modules/videos/models.py index eb31b674..17596e59 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -100,8 +100,12 @@ class Video(ModelHelper, Base): @staticmethod def do_model(file_group: FileGroup, session: Session) -> 'Video': - video = Video(file_group_id=file_group.id, file_group=file_group) - session.add(video) + # Check if a Video already exists for this FileGroup + video = session.query(Video).filter_by(file_group_id=file_group.id).one_or_none() + if not video: + # Create new Video if it doesn't exist + video = Video(file_group_id=file_group.id, file_group=file_group) + session.add(video) video.validate() file_group.indexed = True video.flush() @@ -239,6 +243,15 @@ class Video(ModelHelper, Base): self.channel = channel logger.debug(f'{self} has Channel {channel}') + # Set FileGroup.data with cached file paths for consistency with Archive + self.file_group.data = { + 'id': self.id, + 'video_path': self.video_path, + 'info_json_path': self.info_json_path, + 'poster_path': self.poster_path, + 'caption_paths': self.caption_paths, + } + @staticmethod def from_paths(session: Session, *paths: pathlib.Path) -> 'Video': file_group = FileGroup.from_paths(session, *paths) diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index f9c244a2..8d7a4d76 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -361,3 +361,161 @@ async def test_video_file_format(async_client, test_session, fake_now): request, response = await async_client.post('/api/videos/file_format', json=body) assert response.status_code == HTTPStatus.BAD_REQUEST assert response.json['error'] == 'Filename must end with .%(ext)s' + + +@pytest.mark.asyncio +async def test_video_upload_file_tracking(test_session, async_client, video_factory, await_switches, make_multipart_form, + image_bytes_factory): + """Test that uploading info.json, poster, and caption files via /api/files/upload properly tracks them in Video and FileGroup.""" + from wrolpi.common import get_relative_to_media_directory + + # Step 1: Create a Video with just a video file (no info.json, poster, or captions) + video = video_factory(with_video_file=True, title='file_tracking_test') + test_session.commit() + + # Step 2: Verify Video and FileGroup are created, but optional files do not exist + file_group = video.file_group + assert not video.info_json_path, 'info.json should not exist yet.' + assert not video.poster_path, 'Poster should not exist yet.' + assert not video.caption_paths, 'Captions should not exist yet.' + assert file_group.files is not None, 'FileGroup should have files array' + initial_file_count = len(file_group.files) + + # Step 3: Upload info.json via /api/files/upload + # Extract the video stem (e.g., "file_tracking_test") + video_stem = video.video_path.stem + + # Create info.json content + info_json_data = { + 'title': 'file_tracking_test', + 'duration': 60, + 'uploader': 'test_uploader' + } + info_json_bytes = json.dumps(info_json_data).encode() + + # Get relative destination directory + destination = str(get_relative_to_media_directory(video.video_path.parent)) + filename = f'{video_stem}.info.json' + + # POST to /api/files/upload + forms = [ + dict(name='destination', value=destination), + dict(name='filename', value=filename), + dict(name='chunkNumber', value='0'), + dict(name='totalChunks', value='0'), + dict(name='chunkSize', value=str(len(info_json_bytes))), + dict(name='overwrite', value='false'), + dict(name='chunk', value=info_json_bytes, filename='chunk'), + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}' + + await await_switches() + + # Step 4: Verify info.json is tracked + # Refresh the session to get updated data + test_session.expire_all() + video = test_session.query(Video).filter_by(id=video.id).one() + + # Assert info.json properties exist + assert video.info_json_path is not None, 'Video.info_json_path should exist' + assert video.info_json_file is not None, 'Video.info_json_file should exist' + + # Assert FileGroup has the correct number of files + assert len(video.file_group.files) == initial_file_count + 1, \ + f'FileGroup should have {initial_file_count + 1} files after adding info.json' + assert video.file_group.data.get('info_json_path'), \ + 'Video info.json should be in FileGroup.data' + + # Step 5: Upload poster image and verify it's added to the FileGroup + file_count_before_poster = len(video.file_group.files) + + # Create image using factory + image_data = image_bytes_factory() + + # Upload image with same stem as the Video (using .jpg) + poster_filename = f'{video_stem}.jpg' + forms = [ + dict(name='destination', value=destination), + dict(name='filename', value=poster_filename), + dict(name='chunkNumber', value='0'), + dict(name='totalChunks', value='0'), + dict(name='chunkSize', value=str(len(image_data))), + dict(name='overwrite', value='false'), + dict(name='chunk', value=image_data, filename='chunk'), + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.CREATED, f'Poster upload failed with status {response.status_code}: {response.json}' + + await await_switches() + + # Refresh session to get updated data + test_session.expire_all() + video = test_session.query(Video).filter_by(id=video.id).one() + + # Assert poster properties exist + assert video.poster_path is not None, 'Video.poster_path should exist' + assert video.poster_file is not None, 'Video.poster_file should exist' + assert video.file_group.data.get('poster_path') == video.poster_path, 'Video poster should be in FileGroup.data' + + # Assert poster is in FileGroup.files + file_paths = [f['path'] for f in video.file_group.files] + assert video.poster_path in file_paths, 'Poster should be in FileGroup.files' + + # Assert FileGroup has correct number of files + assert len(video.file_group.files) == file_count_before_poster + 1, \ + f'FileGroup should have {file_count_before_poster + 1} files after adding poster' + + # Step 6: Upload caption file and verify it's added to the FileGroup + file_count_before_caption = len(video.file_group.files) + + # Create caption content + caption_content = """WEBVTT + +00:00:00.000 --> 00:00:02.000 +This is a test caption. + +00:00:02.000 --> 00:00:04.000 +Another line of captions. +""" + + # Upload caption with same stem as the Video + caption_filename = f'{video_stem}.en.vtt' + caption_bytes = caption_content.encode() + forms = [ + dict(name='destination', value=destination), + dict(name='filename', value=caption_filename), + dict(name='chunkNumber', value='0'), + dict(name='totalChunks', value='0'), + dict(name='chunkSize', value=str(len(caption_bytes))), + dict(name='overwrite', value='false'), + dict(name='chunk', value=caption_bytes, filename='chunk'), + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.CREATED, f'Caption upload failed with status {response.status_code}: {response.json}' + + await await_switches() + + # Refresh session to get updated data + test_session.expire_all() + video = test_session.query(Video).filter_by(id=video.id).one() + + # Assert caption properties exist + assert video.caption_paths, 'Video.caption_paths should exist' + assert len(video.caption_paths) == 1, 'Should have exactly one caption file' + assert video.caption_files, 'Video.caption_files should exist' + assert video.file_group.data.get('caption_paths'), 'Video captions should be in FileGroup.data' + + # Assert caption is in FileGroup.files + file_paths = [f['path'] for f in video.file_group.files] + assert video.caption_paths[0] in file_paths, 'Caption should be in FileGroup.files' + + # Assert FileGroup has correct number of files + assert len(video.file_group.files) == file_count_before_caption + 1, \ + f'FileGroup should have {file_count_before_caption + 1} files after adding caption' diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index c01508cb..7b85d283 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -317,7 +317,7 @@ def test_browser_profile_to_yt_dlp_arg(file_name, expected): assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected -def test_video_location(test_session, video_factory, channel_factory): +def test_video_location(test_session, video_factory, channel_factory, async_client): # Video location without a channel, it opens the preview. video1 = video_factory(title='vid1') assert video1.location == '/videos/video/1' diff --git a/modules/zim/lib.py b/modules/zim/lib.py index e8d88b22..3ba8e673 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -43,8 +43,12 @@ __all__ = [ def model_zim(file_group: FileGroup, session: Session) -> Zim: """Create a Zim record for the provided FileGroup.""" - zim = Zim(file_group=file_group, path=file_group.primary_path) - session.add(zim) + # Check if a Zim already exists for this FileGroup + zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none() + if not zim: + # Create new Zim if it doesn't exist + zim = Zim(file_group=file_group, path=file_group.primary_path) + session.add(zim) file_group.title = file_group.primary_path.name file_group.a_text = split_file_name_words(file_group.primary_path.name) file_group.indexed = True diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 11a1cb3b..bd1ddbf9 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -680,8 +680,8 @@ def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.P @pytest.fixture def image_bytes_factory(): - def _() -> bytes: - with tempfile.NamedTemporaryFile(suffix='.png') as fh: + def _(suffix: str = '.png') -> bytes: + with tempfile.NamedTemporaryFile(suffix=suffix) as fh: Image.new('RGB', (1, 1), color='grey').save(fh) fh.seek(0) image = fh.read() diff --git a/wrolpi/files/ebooks.py b/wrolpi/files/ebooks.py index e36541e7..6d9a2b23 100644 --- a/wrolpi/files/ebooks.py +++ b/wrolpi/files/ebooks.py @@ -228,8 +228,12 @@ class EBook(ModelHelper, Base): def model_ebook(file_group: FileGroup, session: Session) -> EBook: """Creates an EBook model based off a File. Searches for its cover in the provided `files`.""" - ebook = EBook(file_group=file_group) - session.add(ebook) + # Check if an EBook already exists for this FileGroup + ebook = session.query(EBook).filter_by(file_group_id=file_group.id).one_or_none() + if not ebook: + # Create new EBook if it doesn't exist + ebook = EBook(file_group=file_group) + session.add(ebook) # Multiple formats may be available. epub_file = next(iter(ebook.file_group.my_epub_files()), None) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 2615385c..756eb5a3 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1582,61 +1582,61 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> logger.warning('upsert_file called, but all files are in ignored directories!') raise IgnoredDirectoryError(f'all files are in ignored directories: {file}') - for i in range(2): - # Try multiple times because uploads happen concurrently and may conflict. - # TODO convert uploads to synchronous in UI. - with get_db_session() as session: - file_group = FileGroup.from_paths(session, *paths) - # Re-index the contents of the file. + # Use a single session context for all database operations + with get_db_session(commit=True) as session: + # Create/update FileGroup with retry logic + file_group = None + for i in range(2): try: - session.flush([file_group, ]) - file_group_id = file_group.id - session.commit() + file_group = FileGroup.from_paths(session, *paths) + session.flush([file_group]) break except IntegrityError: # Another process inserted this FileGroup. logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}') + session.rollback() + if i == 1: + raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}') continue - else: - raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}') - logger.debug(f'upsert_file: {file_group}') - try: - with get_db_session(commit=True) as session: - file_group = FileGroup.find_by_id(file_group_id, session) + if not file_group: + raise RuntimeError(f'upsert_file failed to create FileGroup! {file}') + + logger.debug(f'upsert_file: {file_group}') + + # Model the FileGroup (creates Archive/Video/etc. and sets indexed=True) + try: file_group.do_model(session) - except Exception as e: - logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e) - if PYTEST: - raise + except Exception as e: + logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e) + if PYTEST: + raise - # If user uploads a file, then remove it from the download skip list so comments can be downloaded. Modify the - # download (if any) so that the user can click on it and view the uploaded file. - if url := file_group.url if file_group and file_group.url else None: - if download_manager.is_skipped(url): - download_manager.remove_from_skip_list(url) - with get_db_session() as session: - if download := Download.get_by_url(url, session): + # If user uploads a file, then remove it from the download skip list so comments can be downloaded. + # Modify the download (if any) so that the user can click on it and view the uploaded file. + if file_group.url: + if download_manager.is_skipped(file_group.url): + download_manager.remove_from_skip_list(file_group.url) + if download := Download.get_by_url(file_group.url, session): # Mark download as completed download.complete() # Link the download to the location of the uploaded file. model = file_group.get_model_record() download.location = model.location if model else file_group.location - session.commit() - upsert_directories([], file.parents) - - session.commit() - - if tag_names: - with get_db_session(commit=True) as session: - file_group = FileGroup.find_by_id(file_group_id, session) + # Add tags if provided + if tag_names: for tag_name in tag_names: if tag_name not in file_group.tag_names: - tag = Tag.get_by_name(tag_name) - file_group.add_tag(tag.id) + tag = Tag.get_by_name(tag_name, session) + file_group.add_tag(tag.id, session=session) + + # Commit all changes in one transaction + session.commit() + + # Upsert directories after FileGroup is committed + upsert_directories([], file.parents) - file_group.flush() return file_group diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 489793c4..7eb2fda2 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -513,13 +513,10 @@ class FileGroup(ModelHelper, Base): def do_model(self, session: Session) -> Optional[ModelHelper]: """Get/Create the Model record (Video/Archive/etc.) for this FileGroup, if any.""" if model_class := self.get_model_class(): - if model := model_class.get_by_path(self.primary_path, session): - # Already modeled. - logger.debug(f'already modeled: {self} {model}') - return model - # Create new model (Video/Archive/Ebook). + # Always call do_model() to ensure FileGroup.data is updated when files change. + # The model's do_model() should be idempotent (safe to call multiple times). model = model_class.do_model(self, session) - logger.debug(f'new model: {self} {model}') + logger.debug(f'modeled: {self} {model}') return model # Index the file if no models are available.