mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Increasing logging during refresh. Not setting channel_id in file_group.data in favor of using video.channel_id.
This commit is contained in:
parent
4e3c577202
commit
b5a9e45806
7 changed files with 51 additions and 31 deletions
|
|
@ -6,7 +6,8 @@ from typing import Tuple, List
|
|||
from sqlalchemy import not_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
|
||||
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length, \
|
||||
slow_logger
|
||||
from wrolpi.db import optional_session, get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError, InvalidArchive
|
||||
|
|
@ -69,9 +70,14 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
|||
|
||||
# All Archives have a Singlefile.
|
||||
for file in html_paths:
|
||||
if is_singlefile_file(file):
|
||||
singlefile_path = file
|
||||
break
|
||||
try:
|
||||
if is_singlefile_file(file):
|
||||
singlefile_path = file
|
||||
break
|
||||
except Exception as e:
|
||||
if PYTEST:
|
||||
raise
|
||||
logger.debug(f'Cannot check is_singlefile_file of {repr(file)}', exc_info=e)
|
||||
else:
|
||||
logger.debug(f'No Archive singlefile found in {file_group}')
|
||||
raise InvalidArchive('FileGroup does not contain a singlefile')
|
||||
|
|
@ -147,27 +153,28 @@ async def archive_modeler():
|
|||
.outerjoin(Archive, Archive.file_group_id == FileGroup.id) \
|
||||
.limit(20)
|
||||
|
||||
logger.debug(f'{len(list(invalid_archives))=}')
|
||||
|
||||
processed = 0
|
||||
for file_group, archive in results:
|
||||
processed += 1
|
||||
|
||||
if archive:
|
||||
try:
|
||||
archive_id = archive.id
|
||||
archive.validate()
|
||||
with slow_logger(1, f'Modeling old archive took %(elapsed)s seconds'):
|
||||
archive_id = archive.id
|
||||
archive.validate()
|
||||
except Exception:
|
||||
# Archive could not be indexed.
|
||||
invalid_archives |= {file_group.id, }
|
||||
|
||||
logger.error(f'Unable to validate Archive {archive_id}')
|
||||
if PYTEST:
|
||||
raise
|
||||
else:
|
||||
try:
|
||||
model_archive(file_group, session=session)
|
||||
with slow_logger(1, f'Modeling new archive took %(elapsed)s seconds'):
|
||||
model_archive(file_group, session=session)
|
||||
except InvalidArchive:
|
||||
# FileGroup was not an archive group.
|
||||
invalid_archives |= {file_group.id, }
|
||||
# May not be a real Singlefile archive.
|
||||
pass
|
||||
|
||||
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
|
||||
file_group.indexed = True
|
||||
|
|
@ -176,6 +183,8 @@ async def archive_modeler():
|
|||
# Did not reach limit, do not query again.
|
||||
break
|
||||
|
||||
logger.debug(f'Modeled {processed} Archives')
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ class Archive(Base, ModelHelper):
|
|||
"""Read the Readability JSON file, apply its contents to this record."""
|
||||
readability_json_path = self.readability_json_path
|
||||
if not readability_json_path:
|
||||
logger.warning(f'{self.singlefile_path} does not have an info json file')
|
||||
logger.debug(f'{self.singlefile_path} does not have an info json file')
|
||||
return
|
||||
if not readability_json_path.is_file():
|
||||
error = f'Cannot read data from {readability_json_path} because it not exist.'
|
||||
|
|
|
|||
|
|
@ -456,11 +456,8 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b
|
|||
'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.png'
|
||||
|
||||
|
||||
# @skip_circleci
|
||||
def test_refresh_archives(test_session, test_directory, test_client, make_files_structure):
|
||||
"""
|
||||
Archives can be found and put in the database. A single Archive will have multiple files.
|
||||
"""
|
||||
"""Archives can be found and put in the database. A single Archive will have multiple files."""
|
||||
# The start of a typical singlefile html file.
|
||||
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
|
||||
Page saved with SingleFile
|
||||
|
|
|
|||
|
|
@ -76,14 +76,3 @@ def video_cleanup():
|
|||
v.channel_id IS NULL
|
||||
AND fg.id = v.file_group_id
|
||||
''')
|
||||
curs.execute('''
|
||||
UPDATE file_group fg
|
||||
SET
|
||||
-- Update channel information in file_group.data
|
||||
-- Example: data = {'channel': {'id': 1, 'name': 'example'}}
|
||||
data = fg.data::jsonb || jsonb_build_object('channel',
|
||||
jsonb_build_object('id', v.channel_id, 'name', c.name))
|
||||
FROM video v
|
||||
LEFT JOIN channel c on c.id = v.channel_id
|
||||
WHERE fg.id = v.file_group_id
|
||||
''')
|
||||
|
|
|
|||
|
|
@ -827,7 +827,10 @@ def register_modeler(modeler: callable):
|
|||
|
||||
async def apply_modelers():
|
||||
for modeler in modelers:
|
||||
await modeler()
|
||||
try:
|
||||
await modeler()
|
||||
except Exception as e:
|
||||
logger.error(f'Modeler raised error', exc_info=e)
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
|
@ -921,6 +924,17 @@ def timer(name):
|
|||
logger_.warning(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def slow_logger(max_seconds: int, message: str):
|
||||
before = now()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
elapsed = (now() - before).total_seconds()
|
||||
if elapsed >= max_seconds:
|
||||
logger.warning(message % elapsed)
|
||||
|
||||
|
||||
TIMERS = dict()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -411,9 +411,12 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
|
|||
if isinstance(paths, pathlib.Path):
|
||||
paths = [paths, ]
|
||||
|
||||
refreshing_all_files = False
|
||||
|
||||
with flags.refreshing:
|
||||
if not paths:
|
||||
refresh_logger.warning('Refreshing all files')
|
||||
refreshing_all_files = True
|
||||
else:
|
||||
refresh_logger.warning(f'Refreshing {", ".join(list(map(str, paths)))}')
|
||||
if send_events:
|
||||
|
|
@ -445,12 +448,17 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
|
|||
if send_events:
|
||||
Events.send_global_refresh_completed()
|
||||
refresh_logger.warning('Done refreshing Files')
|
||||
flags.refresh_complete.set()
|
||||
|
||||
if refreshing_all_files:
|
||||
# Only set refresh_complete flag if all files have been refreshed.
|
||||
flags.refresh_complete.set()
|
||||
|
||||
|
||||
async def apply_indexers():
|
||||
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function."""
|
||||
from wrolpi.files.models import FileGroup
|
||||
refresh_logger.info('Applying indexers')
|
||||
|
||||
while True:
|
||||
# Continually query for Files that have not been indexed.
|
||||
with get_db_session(commit=True) as session:
|
||||
|
|
@ -472,6 +480,8 @@ async def apply_indexers():
|
|||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
refresh_logger.debug(f'Indexed {processed} files')
|
||||
|
||||
if processed < 20:
|
||||
# Processed less than the limit, don't do the next query.
|
||||
break
|
||||
|
|
@ -702,6 +712,7 @@ def get_file_statistics():
|
|||
file_group
|
||||
''')
|
||||
statistics = dict(curs.fetchall()[0])
|
||||
statistics['total_count'] = statistics['total_count'] or 0
|
||||
statistics['total_size'] = statistics['total_size'] or 0
|
||||
|
||||
return statistics
|
||||
|
|
|
|||
|
|
@ -259,7 +259,7 @@ def test_file_statistics(test_session, test_client, test_directory, example_pdf,
|
|||
'ebook_count': 2,
|
||||
'image_count': 0,
|
||||
'pdf_count': 1,
|
||||
'total_count': 4,
|
||||
'total_count': 5, # extracted cover
|
||||
'video_count': 1,
|
||||
'zip_count': 0,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue