diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index d36238c5..43acaa10 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -6,7 +6,8 @@ from typing import Tuple, List from sqlalchemy import not_ from sqlalchemy.orm import Session -from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length +from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length, \ + slow_logger from wrolpi.db import optional_session, get_db_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError, InvalidArchive @@ -69,9 +70,14 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive: # All Archives have a Singlefile. for file in html_paths: - if is_singlefile_file(file): - singlefile_path = file - break + try: + if is_singlefile_file(file): + singlefile_path = file + break + except Exception as e: + if PYTEST: + raise + logger.debug(f'Cannot check is_singlefile_file of {repr(file)}', exc_info=e) else: logger.debug(f'No Archive singlefile found in {file_group}') raise InvalidArchive('FileGroup does not contain a singlefile') @@ -147,27 +153,28 @@ async def archive_modeler(): .outerjoin(Archive, Archive.file_group_id == FileGroup.id) \ .limit(20) + logger.debug(f'{len(list(invalid_archives))=}') + processed = 0 for file_group, archive in results: processed += 1 if archive: try: - archive_id = archive.id - archive.validate() + with slow_logger(1, f'Modeling old archive took %(elapsed)s seconds'): + archive_id = archive.id + archive.validate() except Exception: - # Archive could not be indexed. - invalid_archives |= {file_group.id, } - logger.error(f'Unable to validate Archive {archive_id}') if PYTEST: raise else: try: - model_archive(file_group, session=session) + with slow_logger(1, f'Modeling new archive took %(elapsed)s seconds'): + model_archive(file_group, session=session) except InvalidArchive: - # FileGroup was not an archive group. - invalid_archives |= {file_group.id, } + # May not be a real Singlefile archive. + pass # Even if indexing fails, we mark it as indexed. We won't retry indexing this. file_group.indexed = True @@ -176,6 +183,8 @@ async def archive_modeler(): # Did not reach limit, do not query again. break + logger.debug(f'Modeled {processed} Archives') + # Sleep to catch cancel. await asyncio.sleep(0) diff --git a/modules/archive/models.py b/modules/archive/models.py index 9ed2e698..adad4222 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -153,7 +153,7 @@ class Archive(Base, ModelHelper): """Read the Readability JSON file, apply its contents to this record.""" readability_json_path = self.readability_json_path if not readability_json_path: - logger.warning(f'{self.singlefile_path} does not have an info json file') + logger.debug(f'{self.singlefile_path} does not have an info json file') return if not readability_json_path.is_file(): error = f'Cannot read data from {readability_json_path} because it not exist.' diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 1fffbcf8..28e58550 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -456,11 +456,8 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b 'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.png' -# @skip_circleci def test_refresh_archives(test_session, test_directory, test_client, make_files_structure): - """ - Archives can be found and put in the database. A single Archive will have multiple files. - """ + """Archives can be found and put in the database. A single Archive will have multiple files.""" # The start of a typical singlefile html file. singlefile_contents = '''