File.d_text is reformatted to break up long lines.

This commit is contained in:
Roland 2022-12-27 15:41:55 -07:00
parent 35145d0335
commit 95c8610df2
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
9 changed files with 101 additions and 21 deletions

View file

@ -5,7 +5,7 @@ from typing import Tuple, Dict, List
from sqlalchemy.orm import Session
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
from wrolpi.db import optional_session, get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
@ -182,4 +182,5 @@ class ArchiveIndexer(Indexer, ABC):
return None, None, None, None
a = cls.get_title(file)
d = cls.get_article(file)
d = split_lines_by_length(d)
return a, None, None, d

View file

@ -675,7 +675,7 @@ async def test_archive_download_index(test_session, test_directory, image_file):
assert archive.readability_path and archive.readability_path.exists()
assert archive.readability_json_path
assert archive.readability_txt_path and archive.readability_txt_path.exists()
assert archive.readability_txt_file.d_text == '{the,readability}', 'Readability text was not indexed'
assert archive.readability_txt_file.d_text == 'the readability', 'Readability text was not indexed'
assert archive.title == 'the singlefile', 'Did not get the title from the singlefile'
assert archive.screenshot_path and archive.screenshot_file and archive.screenshot_path.is_file(), \
'Did not store the screenshot'

View file

@ -7,7 +7,7 @@ from sqlalchemy.orm import Session
from modules.videos.models import Video, Channel
from wrolpi.captions import read_captions, extract_captions
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
from wrolpi.db import get_db_curs
from wrolpi.files.indexers import Indexer, register_indexer
from wrolpi.files.lib import split_path_stem_and_suffix
@ -159,4 +159,5 @@ class VideoIndexer(Indexer, ABC):
elif EXTRACT_SUBTITLES:
d = extract_captions(file.path) or ''
d = split_lines_by_length(d)
return a, None, c, d