diff --git a/app/src/components/FileBrowser.js b/app/src/components/FileBrowser.js
index e244c5f9..39f3f6b7 100644
--- a/app/src/components/FileBrowser.js
+++ b/app/src/components/FileBrowser.js
@@ -19,7 +19,6 @@ import {SortableTable} from "./SortableTable";
import {useBrowseFiles, useMediaDirectory, useWROLMode} from "../hooks/customHooks";
import {DirectoryRefreshButton, FileRowTagIcon, FilesRefreshButton} from "./Files";
import {FilePreviewContext} from "./FilePreview";
-import {StatusContext} from "../contexts/contexts";
function depthIndentation(path) {
// Repeated spaces for every folder a path is in.
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index cf92fd58..efa4ff4d 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -16,7 +16,8 @@ from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
-from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post
+from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post, \
+ format_html_string, split_lines_by_length
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy
@@ -238,15 +239,18 @@ async def do_archive(url: str) -> Archive:
if readability:
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
with archive_files.readability.open('wt') as fh:
- fh.write(readability.pop('content'))
+ content = format_html_string(readability.pop('content'))
+ fh.write(content)
with archive_files.readability_txt.open('wt') as fh:
readability_txt = readability.pop('textContent')
+ readability_txt = split_lines_by_length(readability_txt)
fh.write(readability_txt)
else:
# No readability was returned, so there are no files.
archive_files.readability_txt = archive_files.readability = None
# Store the single-file HTML in its own file.
+ singlefile = format_html_string(singlefile)
archive_files.singlefile.write_text(singlefile)
if screenshot:
@@ -258,7 +262,7 @@ async def do_archive(url: str) -> Archive:
readability = readability or {}
readability['url'] = url
with archive_files.readability_json.open('wt') as fh:
- fh.write(json.dumps(readability))
+ fh.write(json.dumps(readability, indent=2))
# Create any File models that we can, index them all.
paths = (
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index ecf8591c..67d57026 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -24,7 +24,7 @@ def make_fake_request_archive(readability=True, screenshot=True, title=True):
Image.new('RGB', (1, 1), color='grey').save(fh)
fh.seek(0)
image = fh.read()
- singlefile = '\ntest single-file\nジにてこちら\nsome title'
r = dict(
content=f'test readability content',
textContent='test readability textContent',
@@ -298,15 +298,35 @@ async def test_new_archive(test_session, fake_now):
assert archive1.url is not None
assert archive1.domain is not None
- # The actual files were dumped and read correctly.
- with open(archive1.singlefile_path) as fh:
- assert fh.read() == '
+
+
+ test single-file
+ジにてこちら
+
+
+ some title
+
+
+
+'''
+ assert archive1.readability_path.read_text() == '''
+
+
+ test readability content
+
+
+
+'''
with open(archive1.readability_txt_path) as fh:
assert fh.read() == 'test readability textContent'
- with open(archive1.readability_json_path) as fh:
- assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'}
+ assert archive1.readability_json_path.read_text() == '''{
+ "title": "\\u30b8\\u306b\\u3066\\u3053\\u3061\\u3089",
+ "url": "https://example.com"
+}'''
fake_now(datetime(2000, 1, 2))
archive2 = await do_archive('https://example.com')
@@ -613,7 +633,9 @@ async def test_archive_download_index(test_session, test_directory, image_file):
assert isinstance(archive, lib.Archive), 'Did not get an archive'
assert archive.singlefile_path and archive.singlefile_path.exists() and \
- archive.singlefile_path.read_text() == singlefile, 'Singlefile was not stored'
+ archive.singlefile_path.read_text() == \
+ '\n \n \n the singlefile\n \n \n\n', \
+ 'Singlefile was not formatted'
assert archive.readability_path and archive.readability_path.exists()
assert archive.readability_json_path
assert archive.readability_txt_path and archive.readability_txt_path.exists()
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index b42e11de..4b130141 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -13,7 +13,8 @@ from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE # noqa
from wrolpi.cmd import YT_DLP_BIN
-from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task
+from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
+ format_json_file
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.db import optional_session
@@ -331,6 +332,9 @@ class VideoDownloader(Downloader, ABC):
video.channel_id = channel_id
video_id = video.id
+ if video.info_json_path:
+ format_json_file(video.info_json_path)
+
session.commit()
if download.settings and (tag_names := download.settings.get('tag_names')):
diff --git a/modules/videos/models.py b/modules/videos/models.py
index d0d316dd..a9844237 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -276,7 +276,8 @@ class Video(ModelHelper, Base):
@property
def caption_files(self) -> List[dict]:
- return self.file_group.my_files('text/vtt') + self.file_group.my_files('text/srt')
+ return self.file_group.my_files('text/vtt') + self.file_group.my_files('text/srt') \
+ + self.file_group.my_files('application/x-subrip')
@property
def caption_paths(self) -> List[pathlib.Path]:
diff --git a/wrolpi/common.py b/wrolpi/common.py
index 4eb8bffd..0c5ffe8e 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -1477,3 +1477,45 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
headlines = [tuple(i) for i in curs.fetchall()]
return headlines
+
+
+def format_json_file(file: pathlib.Path, indent: int = 2):
+ """Reformat the contents of a JSON file to include indentation to increase readability."""
+ if not indent or indent <= 0:
+ raise RuntimeError('Ident must be greater than 0')
+ if not file or not file.is_file():
+ raise RuntimeError(f'File does not exist: {file}')
+
+ with file.open('rt') as fh:
+ content = json.load(fh)
+
+ copy = file.with_suffix('.json2')
+ try:
+ with copy.open('wt') as fh:
+ json.dump(content, fh, indent=indent)
+ copy.rename(file)
+ finally:
+ if copy.is_file():
+ copy.unlink()
+
+
+def format_html_string(html: str) -> str:
+ soup = BeautifulSoup(html)
+ return soup.prettify()
+
+
+def format_html_file(file: pathlib.Path):
+ """Reformat the contents of an HTML file to include indentation to increase readability."""
+ if not file or not file.is_file():
+ raise RuntimeError(f'File does not exist: {file}')
+
+ pretty = format_html_string(file.read_text())
+
+ copy = file.with_suffix('.html2')
+ try:
+ with copy.open('wt') as fh:
+ fh.write(pretty)
+ copy.rename(file)
+ finally:
+ if copy.is_file():
+ copy.unlink()
diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py
index 75b42378..1b6bcdfb 100644
--- a/wrolpi/files/test/test_lib.py
+++ b/wrolpi/files/test/test_lib.py
@@ -183,7 +183,7 @@ async def test__upsert_files(test_session, make_files_structure, test_directory,
assert_file_groups([
{'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': False,
'files': [
- {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
+ {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'application/x-subrip'},
{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
]},
{'primary_path': bar, 'idempotency': idempotency, 'indexed': False,
@@ -210,7 +210,7 @@ async def test__upsert_files(test_session, make_files_structure, test_directory,
assert_file_groups([
{'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True,
'files': [
- {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
+ {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'application/x-subrip'},
{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
]},
{'primary_path': bar, 'idempotency': idempotency, 'indexed': False,
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index 0c4d773b..3048718f 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -1,5 +1,6 @@
import asyncio
import ctypes.wintypes
+import json
import multiprocessing
import os
import pathlib
@@ -719,3 +720,65 @@ Paragraph outside an element.'''
)
def test_human_bandwidth(bps, expected):
assert common.human_bandwidth(bps) == expected
+
+
+def test_format_json_file():
+ with tempfile.TemporaryDirectory() as d:
+ with tempfile.NamedTemporaryFile(suffix='.json', dir=d) as fh:
+ file = pathlib.Path(fh.name)
+ file.write_text(json.dumps({
+ 'one': 1,
+ 'two': 2,
+ 'three': 3,
+ 'four': 4,
+ 'five': 5,
+ 'six': 6,
+ 'seven': 7,
+ 'eight': 8,
+ 'nine': 9,
+ 'ten': 10,
+ }))
+ # Only one file was created.
+ assert len(list(file.parent.iterdir())) == 1
+
+ unformatted = '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10}'
+ assert file.read_text() == unformatted
+
+ common.format_json_file(file)
+ assert file.read_text() == '''{
+ "one": 1,
+ "two": 2,
+ "three": 3,
+ "four": 4,
+ "five": 5,
+ "six": 6,
+ "seven": 7,
+ "eight": 8,
+ "nine": 9,
+ "ten": 10
+}'''
+ # The copy was deleted.
+ assert len(list(file.parent.iterdir())) == 1
+
+
+def test_format_html_file():
+ with tempfile.NamedTemporaryFile() as fh:
+ file = pathlib.Path(fh.name)
+ unformatted = 'TitleBody'
+ file.write_text(unformatted)
+
+ assert file.read_text() == unformatted
+
+ common.format_html_file(file)
+ assert file.read_text() == '''
+
+
+
+ Title
+
+
+
+ Body
+
+
+'''