mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
File.d_text is reformatted to break up long lines.
This commit is contained in:
Roland
parent
35145d0335
commit
95c8610df2
9 changed files with 101 additions and 21 deletions
|
|
@ -5,7 +5,7 @@ from typing import Tuple, Dict, List
|
|||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent
|
||||
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
|
||||
from wrolpi.db import optional_session, get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
|
|
@ -182,4 +182,5 @@ class ArchiveIndexer(Indexer, ABC):
|
|||
return None, None, None, None
|
||||
a = cls.get_title(file)
|
||||
d = cls.get_article(file)
|
||||
d = split_lines_by_length(d)
|
||||
return a, None, None, d
|
||||
|
|
|
|||
|
|
@ -675,7 +675,7 @@ async def test_archive_download_index(test_session, test_directory, image_file):
|
|||
assert archive.readability_path and archive.readability_path.exists()
|
||||
assert archive.readability_json_path
|
||||
assert archive.readability_txt_path and archive.readability_txt_path.exists()
|
||||
assert archive.readability_txt_file.d_text == '{the,readability}', 'Readability text was not indexed'
|
||||
assert archive.readability_txt_file.d_text == 'the readability', 'Readability text was not indexed'
|
||||
assert archive.title == 'the singlefile', 'Did not get the title from the singlefile'
|
||||
assert archive.screenshot_path and archive.screenshot_file and archive.screenshot_path.is_file(), \
|
||||
'Did not store the screenshot'
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.captions import read_captions, extract_captions
|
||||
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent
|
||||
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length
|
||||
from wrolpi.db import get_db_curs
|
||||
from wrolpi.files.indexers import Indexer, register_indexer
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
|
|
@ -159,4 +159,5 @@ class VideoIndexer(Indexer, ABC):
|
|||
elif EXTRACT_SUBTITLES:
|
||||
d = extract_captions(file.path) or ''
|
||||
|
||||
d = split_lines_by_length(d)
|
||||
return a, None, c, d
|
||||
|
|
|
|||
BIN
test/example.pdf
BIN
test/example.pdf
Binary file not shown.
|
|
@ -975,7 +975,7 @@ def limit_concurrent(limit: int, throw: bool = False):
|
|||
return wrapper
|
||||
|
||||
|
||||
def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> List[str]:
|
||||
def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> Union[List[str], str]:
|
||||
"""
|
||||
Shorten an object. This is useful when inserting something into a tsvector.
|
||||
|
||||
|
|
@ -1082,3 +1082,41 @@ def cancelable_wrapper(func: callable):
|
|||
|
||||
return wrapped
|
||||
|
||||
|
||||
WHITESPACE_SPLITTER = re.compile(r'\s+')
|
||||
TAB = re.compile(r'\t+')
|
||||
|
||||
|
||||
def split_lines_by_length(text: str, max_line_length: int = 38) -> str:
|
||||
"""
|
||||
Break up any long lines along word boundaries.
|
||||
|
||||
Newlines are preserved, but whitspace (spaces/tabs) is replaced with one space.
|
||||
|
||||
@param max_line_length: The most amount of characters a line will contain. 38 is default for mobile vertical.
|
||||
|
||||
>>> split_lines_by_length('Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do')
|
||||
'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do'
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
text = TAB.sub(' ', text)
|
||||
new_text = ''
|
||||
for line in text.splitlines():
|
||||
if len(line) > max_line_length:
|
||||
# This line is too long, break it up.
|
||||
new_line = ''
|
||||
words = WHITESPACE_SPLITTER.split(line)
|
||||
while words:
|
||||
word = words.pop(0)
|
||||
if len(possible_line := f'{new_line} {word}') <= max_line_length:
|
||||
new_line = possible_line
|
||||
else:
|
||||
# New word makes the line too long, start a new line.
|
||||
new_text += f'\n{new_line.strip()}'
|
||||
new_line = word
|
||||
new_text += f'\n{new_line}'
|
||||
else:
|
||||
new_text += f'\n{line}'
|
||||
return new_text.lstrip('\n')
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import dataclasses
|
||||
import pathlib
|
||||
import re
|
||||
from abc import ABC
|
||||
from collections import defaultdict
|
||||
from typing import List, Type, Tuple, Generator
|
||||
|
|
@ -13,7 +12,7 @@ try:
|
|||
except ImportError:
|
||||
PdfReader = None
|
||||
|
||||
from wrolpi.common import logger, truncate_generator_bytes
|
||||
from wrolpi.common import logger, truncate_generator_bytes, split_lines_by_length, truncate_object_bytes
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -91,6 +90,7 @@ class ZipIndexer(Indexer, ABC):
|
|||
try:
|
||||
with ZipFile(path, 'r') as zip_:
|
||||
file_names = [' '.join(split_file_name_words(pathlib.Path(name).name)) for name in zip_.namelist()]
|
||||
file_names = truncate_object_bytes(file_names, FILE_MAX_TEXT_SIZE)
|
||||
return file_names
|
||||
except Exception as e:
|
||||
logger.error(f'Unable to get information from zip: {path}', exc_info=e)
|
||||
|
|
@ -98,9 +98,6 @@ class ZipIndexer(Indexer, ABC):
|
|||
raise
|
||||
|
||||
|
||||
WORD_SPLITTER = re.compile(r"([\w’'-]+)")
|
||||
|
||||
|
||||
@register_indexer('text/plain')
|
||||
class TextIndexer(Indexer, ABC):
|
||||
"""Handles plain text files.
|
||||
|
|
@ -115,11 +112,12 @@ class TextIndexer(Indexer, ABC):
|
|||
return a, None, None, words
|
||||
|
||||
@staticmethod
|
||||
def get_words(path: pathlib.Path) -> List[str]:
|
||||
def get_words(path: pathlib.Path) -> str:
|
||||
"""Read the words from a file."""
|
||||
# TODO this only supports English.
|
||||
contents = path.read_text()
|
||||
words = WORD_SPLITTER.findall(contents)
|
||||
contents = truncate_object_bytes(contents, FILE_MAX_TEXT_SIZE)
|
||||
words = split_lines_by_length(contents)
|
||||
return words
|
||||
|
||||
|
||||
|
|
@ -153,6 +151,8 @@ class PDFIndexer(Indexer, ABC):
|
|||
try:
|
||||
# PDFs are complex, don't fail to create title index if text extraction fails.
|
||||
words = '\n'.join(truncate_generator_bytes(cls.get_words(reader, path), FILE_MAX_TEXT_SIZE))
|
||||
words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE)
|
||||
words = split_lines_by_length(words)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to index {path}', exc_info=e)
|
||||
if PYTEST:
|
||||
|
|
|
|||
|
|
@ -103,11 +103,7 @@ class File(ModelHelper, Base):
|
|||
indexer = self.indexer
|
||||
# Only read the contents of the file once.
|
||||
start = now()
|
||||
a_text, b_text, c_text, d_text = indexer.create_index(self)
|
||||
self.a_text = truncate_object_bytes(a_text, FILE_MAX_TEXT_SIZE)
|
||||
self.b_text = truncate_object_bytes(b_text, FILE_MAX_TEXT_SIZE)
|
||||
self.c_text = truncate_object_bytes(c_text, FILE_MAX_TEXT_SIZE)
|
||||
self.d_text = truncate_object_bytes(d_text, FILE_MAX_TEXT_SIZE)
|
||||
self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self)
|
||||
if (total_seconds := (now() - start).total_seconds()) > 1:
|
||||
logger.info(f'Indexing {self.path} took {total_seconds} seconds')
|
||||
|
||||
|
|
|
|||
|
|
@ -428,7 +428,7 @@ async def test_large_text_indexer(test_session, make_files_structure):
|
|||
|
||||
large_file: File = test_session.query(File).one()
|
||||
assert len(large_file.d_text) < large.stat().st_size
|
||||
assert len(large_file.d_text) == 46_117
|
||||
assert len(large_file.d_text) == 90_072
|
||||
|
||||
|
||||
def test_glob_shared_stem(make_files_structure):
|
||||
|
|
@ -499,8 +499,23 @@ def test_pdf_indexer(example_pdf):
|
|||
assert b_text == 'roland'
|
||||
# The parsed file name.
|
||||
assert c_text == ['example', 'pdf']
|
||||
# Both pages of the example PDF are extracted.
|
||||
assert d_text == 'Page one\nPage two'
|
||||
# All pages are extracted. Text is formatted to fit on vertical screen.
|
||||
assert d_text == 'Page one\n' \
|
||||
'Page two\n' \
|
||||
'Lorem ipsum dolor sit amet,\n' \
|
||||
'consectetur adipiscing elit, sed do\n' \
|
||||
'eiusmod tempor incididunt ut labore et\n' \
|
||||
'\n' \
|
||||
'dolore magna aliqua. Ut enim ad minim\n' \
|
||||
'veniam, quis nostrud exercitation\n' \
|
||||
'ullamco laboris nisi ut \n' \
|
||||
'aliquip ex ea commodo consequat. Duis\n' \
|
||||
'aute irure dolor in reprehenderit in\n' \
|
||||
'voluptate velit esse cillum \n' \
|
||||
'dolore eu fugiat nulla pariatur.\n' \
|
||||
'Excepteur sint occaecat cupidatat non\n' \
|
||||
'proident, sunt in culpa qui officia \n' \
|
||||
'deserunt mollit anim id est laborum.'
|
||||
|
||||
|
||||
def test_pdf_indexer_max_size(example_pdf):
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@ import pytz
|
|||
import wrolpi.vars
|
||||
from wrolpi import common
|
||||
from wrolpi.common import cum_timer, TIMERS, print_timer, limit_concurrent, run_after
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.errors import InvalidTimezone
|
||||
from wrolpi.test.common import build_test_directories
|
||||
|
||||
|
||||
|
|
@ -567,3 +565,34 @@ async def test_run_after():
|
|||
# Sleep so "after" will run.
|
||||
await asyncio.sleep(0)
|
||||
assert count.value == 2, 'Counter did not run after'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'max_line_length,text,expected', [
|
||||
(38, '', ''),
|
||||
(38, 'Lorem', 'Lorem'),
|
||||
(
|
||||
38,
|
||||
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do',
|
||||
'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do'
|
||||
),
|
||||
(
|
||||
10,
|
||||
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do',
|
||||
'Lorem\nipsum\ndolor sit\namet,\nconsectetur\nadipiscing\nelit, sed\ndo'
|
||||
),
|
||||
(
|
||||
20,
|
||||
'Lorem\nipsum\ndolor sit amet,\t\tconsectetur\n\nadipiscing elit, sed do',
|
||||
'Lorem\nipsum\ndolor sit amet,\nconsectetur\n\nadipiscing elit,\nsed do'
|
||||
),
|
||||
(
|
||||
38,
|
||||
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.',
|
||||
'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do\neiusmod tempor incididunt ut labore et\ndolore magna aliqua.'
|
||||
),
|
||||
(38, 'long-word-which-should-not-be-split-up', 'long-word-which-should-not-be-split-up'),
|
||||
]
|
||||
)
|
||||
def test_split_lines_by_length(max_line_length, text, expected):
|
||||
assert common.split_lines_by_length(text, max_line_length=max_line_length) == expected
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue