Increasing logging during refresh. Not setting channel_id in file_group.data in favor of using video.channel_id.

This commit is contained in:
Roland 2023-03-27 10:02:47 -06:00
parent 4e3c577202
commit b5a9e45806
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
7 changed files with 51 additions and 31 deletions

View file

@ -6,7 +6,8 @@ from typing import Tuple, List
from sqlalchemy import not_
from sqlalchemy.orm import Session
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length, \
slow_logger
from wrolpi.db import optional_session, get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError, InvalidArchive
@ -69,9 +70,14 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
# All Archives have a Singlefile.
for file in html_paths:
if is_singlefile_file(file):
singlefile_path = file
break
try:
if is_singlefile_file(file):
singlefile_path = file
break
except Exception as e:
if PYTEST:
raise
logger.debug(f'Cannot check is_singlefile_file of {repr(file)}', exc_info=e)
else:
logger.debug(f'No Archive singlefile found in {file_group}')
raise InvalidArchive('FileGroup does not contain a singlefile')
@ -147,27 +153,28 @@ async def archive_modeler():
.outerjoin(Archive, Archive.file_group_id == FileGroup.id) \
.limit(20)
logger.debug(f'{len(list(invalid_archives))=}')
processed = 0
for file_group, archive in results:
processed += 1
if archive:
try:
archive_id = archive.id
archive.validate()
with slow_logger(1, f'Modeling old archive took %(elapsed)s seconds'):
archive_id = archive.id
archive.validate()
except Exception:
# Archive could not be indexed.
invalid_archives |= {file_group.id, }
logger.error(f'Unable to validate Archive {archive_id}')
if PYTEST:
raise
else:
try:
model_archive(file_group, session=session)
with slow_logger(1, f'Modeling new archive took %(elapsed)s seconds'):
model_archive(file_group, session=session)
except InvalidArchive:
# FileGroup was not an archive group.
invalid_archives |= {file_group.id, }
# May not be a real Singlefile archive.
pass
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
file_group.indexed = True
@ -176,6 +183,8 @@ async def archive_modeler():
# Did not reach limit, do not query again.
break
logger.debug(f'Modeled {processed} Archives')
# Sleep to catch cancel.
await asyncio.sleep(0)

View file

@ -153,7 +153,7 @@ class Archive(Base, ModelHelper):
"""Read the Readability JSON file, apply its contents to this record."""
readability_json_path = self.readability_json_path
if not readability_json_path:
logger.warning(f'{self.singlefile_path} does not have an info json file')
logger.debug(f'{self.singlefile_path} does not have an info json file')
return
if not readability_json_path.is_file():
error = f'Cannot read data from {readability_json_path} because it not exist.'

View file

@ -456,11 +456,8 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b
'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.png'
# @skip_circleci
def test_refresh_archives(test_session, test_directory, test_client, make_files_structure):
"""
Archives can be found and put in the database. A single Archive will have multiple files.
"""
"""Archives can be found and put in the database. A single Archive will have multiple files."""
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile

View file

@ -76,14 +76,3 @@ def video_cleanup():
v.channel_id IS NULL
AND fg.id = v.file_group_id
''')
curs.execute('''
UPDATE file_group fg
SET
-- Update channel information in file_group.data
-- Example: data = {'channel': {'id': 1, 'name': 'example'}}
data = fg.data::jsonb || jsonb_build_object('channel',
jsonb_build_object('id', v.channel_id, 'name', c.name))
FROM video v
LEFT JOIN channel c on c.id = v.channel_id
WHERE fg.id = v.file_group_id
''')

View file

@ -827,7 +827,10 @@ def register_modeler(modeler: callable):
async def apply_modelers():
for modeler in modelers:
await modeler()
try:
await modeler()
except Exception as e:
logger.error(f'Modeler raised error', exc_info=e)
# Sleep to catch cancel.
await asyncio.sleep(0)
@ -921,6 +924,17 @@ def timer(name):
logger_.warning(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
@contextlib.contextmanager
def slow_logger(max_seconds: int, message: str):
before = now()
try:
yield
finally:
elapsed = (now() - before).total_seconds()
if elapsed >= max_seconds:
logger.warning(message % elapsed)
TIMERS = dict()

View file

@ -411,9 +411,12 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
if isinstance(paths, pathlib.Path):
paths = [paths, ]
refreshing_all_files = False
with flags.refreshing:
if not paths:
refresh_logger.warning('Refreshing all files')
refreshing_all_files = True
else:
refresh_logger.warning(f'Refreshing {", ".join(list(map(str, paths)))}')
if send_events:
@ -445,12 +448,17 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
if send_events:
Events.send_global_refresh_completed()
refresh_logger.warning('Done refreshing Files')
flags.refresh_complete.set()
if refreshing_all_files:
# Only set refresh_complete flag if all files have been refreshed.
flags.refresh_complete.set()
async def apply_indexers():
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function."""
from wrolpi.files.models import FileGroup
refresh_logger.info('Applying indexers')
while True:
# Continually query for Files that have not been indexed.
with get_db_session(commit=True) as session:
@ -472,6 +480,8 @@ async def apply_indexers():
# Sleep to catch cancel.
await asyncio.sleep(0)
refresh_logger.debug(f'Indexed {processed} files')
if processed < 20:
# Processed less than the limit, don't do the next query.
break
@ -702,6 +712,7 @@ def get_file_statistics():
file_group
''')
statistics = dict(curs.fetchall()[0])
statistics['total_count'] = statistics['total_count'] or 0
statistics['total_size'] = statistics['total_size'] or 0
return statistics

View file

@ -259,7 +259,7 @@ def test_file_statistics(test_session, test_client, test_directory, example_pdf,
'ebook_count': 2,
'image_count': 0,
'pdf_count': 1,
'total_count': 4,
'total_count': 5, # extracted cover
'video_count': 1,
'zip_count': 0,
}