diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index dfb8a27f..4307fbb6 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -5,7 +5,7 @@ from typing import Tuple, Dict, List from sqlalchemy.orm import Session -from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent +from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length from wrolpi.db import optional_session, get_db_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError @@ -182,4 +182,5 @@ class ArchiveIndexer(Indexer, ABC): return None, None, None, None a = cls.get_title(file) d = cls.get_article(file) + d = split_lines_by_length(d) return a, None, None, d diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 70d3c73c..55d6b200 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -675,7 +675,7 @@ async def test_archive_download_index(test_session, test_directory, image_file): assert archive.readability_path and archive.readability_path.exists() assert archive.readability_json_path assert archive.readability_txt_path and archive.readability_txt_path.exists() - assert archive.readability_txt_file.d_text == '{the,readability}', 'Readability text was not indexed' + assert archive.readability_txt_file.d_text == 'the readability', 'Readability text was not indexed' assert archive.title == 'the singlefile', 'Did not get the title from the singlefile' assert archive.screenshot_path and archive.screenshot_file and archive.screenshot_path.is_file(), \ 'Did not store the screenshot' diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index 46d084f0..9830e573 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import Session from modules.videos.models import Video, Channel from wrolpi.captions import read_captions, extract_captions -from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent +from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length from wrolpi.db import get_db_curs from wrolpi.files.indexers import Indexer, register_indexer from wrolpi.files.lib import split_path_stem_and_suffix @@ -159,4 +159,5 @@ class VideoIndexer(Indexer, ABC): elif EXTRACT_SUBTITLES: d = extract_captions(file.path) or '' + d = split_lines_by_length(d) return a, None, c, d diff --git a/test/example.pdf b/test/example.pdf index bbcda4bb..4db0cbbf 100644 Binary files a/test/example.pdf and b/test/example.pdf differ diff --git a/wrolpi/common.py b/wrolpi/common.py index adc073e7..e4c9e6c2 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -975,7 +975,7 @@ def limit_concurrent(limit: int, throw: bool = False): return wrapper -def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> List[str]: +def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> Union[List[str], str]: """ Shorten an object. This is useful when inserting something into a tsvector. @@ -1082,3 +1082,41 @@ def cancelable_wrapper(func: callable): return wrapped + +WHITESPACE_SPLITTER = re.compile(r'\s+') +TAB = re.compile(r'\t+') + + +def split_lines_by_length(text: str, max_line_length: int = 38) -> str: + """ + Break up any long lines along word boundaries. + + Newlines are preserved, but whitspace (spaces/tabs) is replaced with one space. + + @param max_line_length: The most amount of characters a line will contain. 38 is default for mobile vertical. + + >>> split_lines_by_length('Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do') + 'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do' + """ + if not text: + return text + + text = TAB.sub(' ', text) + new_text = '' + for line in text.splitlines(): + if len(line) > max_line_length: + # This line is too long, break it up. + new_line = '' + words = WHITESPACE_SPLITTER.split(line) + while words: + word = words.pop(0) + if len(possible_line := f'{new_line} {word}') <= max_line_length: + new_line = possible_line + else: + # New word makes the line too long, start a new line. + new_text += f'\n{new_line.strip()}' + new_line = word + new_text += f'\n{new_line}' + else: + new_text += f'\n{line}' + return new_text.lstrip('\n') diff --git a/wrolpi/files/indexers.py b/wrolpi/files/indexers.py index 9fd0819e..7b23a581 100644 --- a/wrolpi/files/indexers.py +++ b/wrolpi/files/indexers.py @@ -1,6 +1,5 @@ import dataclasses import pathlib -import re from abc import ABC from collections import defaultdict from typing import List, Type, Tuple, Generator @@ -13,7 +12,7 @@ try: except ImportError: PdfReader = None -from wrolpi.common import logger, truncate_generator_bytes +from wrolpi.common import logger, truncate_generator_bytes, split_lines_by_length, truncate_object_bytes logger = logger.getChild(__name__) @@ -91,6 +90,7 @@ class ZipIndexer(Indexer, ABC): try: with ZipFile(path, 'r') as zip_: file_names = [' '.join(split_file_name_words(pathlib.Path(name).name)) for name in zip_.namelist()] + file_names = truncate_object_bytes(file_names, FILE_MAX_TEXT_SIZE) return file_names except Exception as e: logger.error(f'Unable to get information from zip: {path}', exc_info=e) @@ -98,9 +98,6 @@ class ZipIndexer(Indexer, ABC): raise -WORD_SPLITTER = re.compile(r"([\w’'-]+)") - - @register_indexer('text/plain') class TextIndexer(Indexer, ABC): """Handles plain text files. @@ -115,11 +112,12 @@ class TextIndexer(Indexer, ABC): return a, None, None, words @staticmethod - def get_words(path: pathlib.Path) -> List[str]: + def get_words(path: pathlib.Path) -> str: """Read the words from a file.""" # TODO this only supports English. contents = path.read_text() - words = WORD_SPLITTER.findall(contents) + contents = truncate_object_bytes(contents, FILE_MAX_TEXT_SIZE) + words = split_lines_by_length(contents) return words @@ -153,6 +151,8 @@ class PDFIndexer(Indexer, ABC): try: # PDFs are complex, don't fail to create title index if text extraction fails. words = '\n'.join(truncate_generator_bytes(cls.get_words(reader, path), FILE_MAX_TEXT_SIZE)) + words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE) + words = split_lines_by_length(words) except Exception as e: logger.error(f'Failed to index {path}', exc_info=e) if PYTEST: diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 4ef5a06e..000418da 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -103,11 +103,7 @@ class File(ModelHelper, Base): indexer = self.indexer # Only read the contents of the file once. start = now() - a_text, b_text, c_text, d_text = indexer.create_index(self) - self.a_text = truncate_object_bytes(a_text, FILE_MAX_TEXT_SIZE) - self.b_text = truncate_object_bytes(b_text, FILE_MAX_TEXT_SIZE) - self.c_text = truncate_object_bytes(c_text, FILE_MAX_TEXT_SIZE) - self.d_text = truncate_object_bytes(d_text, FILE_MAX_TEXT_SIZE) + self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self) if (total_seconds := (now() - start).total_seconds()) > 1: logger.info(f'Indexing {self.path} took {total_seconds} seconds') diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index d4859157..5b681ae1 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -428,7 +428,7 @@ async def test_large_text_indexer(test_session, make_files_structure): large_file: File = test_session.query(File).one() assert len(large_file.d_text) < large.stat().st_size - assert len(large_file.d_text) == 46_117 + assert len(large_file.d_text) == 90_072 def test_glob_shared_stem(make_files_structure): @@ -499,8 +499,23 @@ def test_pdf_indexer(example_pdf): assert b_text == 'roland' # The parsed file name. assert c_text == ['example', 'pdf'] - # Both pages of the example PDF are extracted. - assert d_text == 'Page one\nPage two' + # All pages are extracted. Text is formatted to fit on vertical screen. + assert d_text == 'Page one\n' \ + 'Page two\n' \ + 'Lorem ipsum dolor sit amet,\n' \ + 'consectetur adipiscing elit, sed do\n' \ + 'eiusmod tempor incididunt ut labore et\n' \ + '\n' \ + 'dolore magna aliqua. Ut enim ad minim\n' \ + 'veniam, quis nostrud exercitation\n' \ + 'ullamco laboris nisi ut \n' \ + 'aliquip ex ea commodo consequat. Duis\n' \ + 'aute irure dolor in reprehenderit in\n' \ + 'voluptate velit esse cillum \n' \ + 'dolore eu fugiat nulla pariatur.\n' \ + 'Excepteur sint occaecat cupidatat non\n' \ + 'proident, sunt in culpa qui officia \n' \ + 'deserunt mollit anim id est laborum.' def test_pdf_indexer_max_size(example_pdf): diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 57e5780d..a03fe3ef 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -17,8 +17,6 @@ import pytz import wrolpi.vars from wrolpi import common from wrolpi.common import cum_timer, TIMERS, print_timer, limit_concurrent, run_after -from wrolpi.dates import now -from wrolpi.errors import InvalidTimezone from wrolpi.test.common import build_test_directories @@ -567,3 +565,34 @@ async def test_run_after(): # Sleep so "after" will run. await asyncio.sleep(0) assert count.value == 2, 'Counter did not run after' + + +@pytest.mark.parametrize( + 'max_line_length,text,expected', [ + (38, '', ''), + (38, 'Lorem', 'Lorem'), + ( + 38, + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do', + 'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do' + ), + ( + 10, + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do', + 'Lorem\nipsum\ndolor sit\namet,\nconsectetur\nadipiscing\nelit, sed\ndo' + ), + ( + 20, + 'Lorem\nipsum\ndolor sit amet,\t\tconsectetur\n\nadipiscing elit, sed do', + 'Lorem\nipsum\ndolor sit amet,\nconsectetur\n\nadipiscing elit,\nsed do' + ), + ( + 38, + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', + 'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do\neiusmod tempor incididunt ut labore et\ndolore magna aliqua.' + ), + (38, 'long-word-which-should-not-be-split-up', 'long-word-which-should-not-be-split-up'), + ] +) +def test_split_lines_by_length(max_line_length, text, expected): + assert common.split_lines_by_length(text, max_line_length=max_line_length) == expected