HTML and JSON files are reformatted when downloaded.

app/src/components/FileBrowser.js

@@ -19,7 +19,6 @@ import {SortableTable} from "./SortableTable";
 import {useBrowseFiles, useMediaDirectory, useWROLMode} from "../hooks/customHooks";
 import {DirectoryRefreshButton, FileRowTagIcon, FilesRefreshButton} from "./Files";
 import {FilePreviewContext} from "./FilePreview";
-import {StatusContext} from "../contexts/contexts";
 
 function depthIndentation(path) {
     // Repeated spaces for every folder a path is in.

modules/archive/lib.py

@@ -16,7 +16,8 @@ from sqlalchemy.orm import Session
 
 from modules.archive.models import Domain, Archive
 from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
-from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post
+from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post, \
+    format_html_string, split_lines_by_length
 from wrolpi.dates import now, Seconds
 from wrolpi.db import get_db_session, get_db_curs, optional_session
 from wrolpi.errors import UnknownArchive, InvalidOrderBy
@@ -238,15 +239,18 @@ async def do_archive(url: str) -> Archive:
     if readability:
         # Write the readability parts to their own files.  Write what is left after pops to the JSON file.
         with archive_files.readability.open('wt') as fh:
-            fh.write(readability.pop('content'))
+            content = format_html_string(readability.pop('content'))
+            fh.write(content)
         with archive_files.readability_txt.open('wt') as fh:
             readability_txt = readability.pop('textContent')
+            readability_txt = split_lines_by_length(readability_txt)
             fh.write(readability_txt)
     else:
         # No readability was returned, so there are no files.
         archive_files.readability_txt = archive_files.readability = None
 
     # Store the single-file HTML in its own file.
+    singlefile = format_html_string(singlefile)
     archive_files.singlefile.write_text(singlefile)
 
     if screenshot:
@@ -258,7 +262,7 @@ async def do_archive(url: str) -> Archive:
     readability = readability or {}
     readability['url'] = url
     with archive_files.readability_json.open('wt') as fh:
-        fh.write(json.dumps(readability))
+        fh.write(json.dumps(readability, indent=2))
 
     # Create any File models that we can, index them all.
     paths = (

modules/archive/test/test_lib.py

@@ -24,7 +24,7 @@ def make_fake_request_archive(readability=True, screenshot=True, title=True):
             Image.new('RGB', (1, 1), color='grey').save(fh)
             fh.seek(0)
             image = fh.read()
-        singlefile = '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
+        singlefile = '<html><!--\n Page saved with SingleFile-->\ntest single-file\nジにてこちら\n<title>some title</title></html>'
         r = dict(
             content=f'<html>test readability content</html>',
             textContent='test readability textContent',
@@ -298,15 +298,35 @@ async def test_new_archive(test_session, fake_now):
         assert archive1.url is not None
         assert archive1.domain is not None
 
-        # The actual files were dumped and read correctly.
-        with open(archive1.singlefile_path) as fh:
-            assert fh.read() == '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
-        with open(archive1.readability_path) as fh:
-            assert fh.read() == '<html>test readability content</html>'
+        # The actual files were dumped and read correctly.  The HTML/JSON files are reformatted.
+        assert archive1.singlefile_path.read_text() == '''<html>
+ <!--
+ Page saved with SingleFile-->
+ <body>
+  <p>
+   test single-file
+ジにてこちら
+  </p>
+  <title>
+   some title
+  </title>
+ </body>
+</html>
+'''
+        assert archive1.readability_path.read_text() == '''<html>
+ <body>
+  <p>
+   test readability content
+  </p>
+ </body>
+</html>
+'''
         with open(archive1.readability_txt_path) as fh:
             assert fh.read() == 'test readability textContent'
-        with open(archive1.readability_json_path) as fh:
-            assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'}
+        assert archive1.readability_json_path.read_text() == '''{
+  "title": "\\u30b8\\u306b\\u3066\\u3053\\u3061\\u3089",
+  "url": "https://example.com"
+}'''
 
         fake_now(datetime(2000, 1, 2))
         archive2 = await do_archive('https://example.com')
@@ -613,7 +633,9 @@ async def test_archive_download_index(test_session, test_directory, image_file):
 
     assert isinstance(archive, lib.Archive), 'Did not get an archive'
     assert archive.singlefile_path and archive.singlefile_path.exists() and \
-           archive.singlefile_path.read_text() == singlefile, 'Singlefile was not stored'
+           archive.singlefile_path.read_text() == \
+           '<html>\n <head>\n  <title>\n   the singlefile\n  </title>\n </head>\n</html>\n', \
+        'Singlefile was not formatted'
     assert archive.readability_path and archive.readability_path.exists()
     assert archive.readability_json_path
     assert archive.readability_txt_path and archive.readability_txt_path.exists()

modules/videos/downloader.py

@@ -13,7 +13,8 @@ from yt_dlp import YoutubeDL
 from yt_dlp.extractor import YoutubeTabIE  # noqa
 
 from wrolpi.cmd import YT_DLP_BIN
-from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task
+from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
+    format_json_file
 from wrolpi.dates import now
 from wrolpi.db import get_db_session
 from wrolpi.db import optional_session
@@ -331,6 +332,9 @@ class VideoDownloader(Downloader, ABC):
                 video.channel_id = channel_id
                 video_id = video.id
 
+                if video.info_json_path:
+                    format_json_file(video.info_json_path)
+
                 session.commit()
 
                 if download.settings and (tag_names := download.settings.get('tag_names')):

modules/videos/models.py

@@ -276,7 +276,8 @@ class Video(ModelHelper, Base):
 
     @property
     def caption_files(self) -> List[dict]:
-        return self.file_group.my_files('text/vtt') + self.file_group.my_files('text/srt')
+        return self.file_group.my_files('text/vtt') + self.file_group.my_files('text/srt') \
+            + self.file_group.my_files('application/x-subrip')
 
     @property
     def caption_paths(self) -> List[pathlib.Path]:

wrolpi/common.py

@@ -1477,3 +1477,45 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
         headlines = [tuple(i) for i in curs.fetchall()]
 
     return headlines
+
+
+def format_json_file(file: pathlib.Path, indent: int = 2):
+    """Reformat the contents of a JSON file to include indentation to increase readability."""
+    if not indent or indent <= 0:
+        raise RuntimeError('Ident must be greater than 0')
+    if not file or not file.is_file():
+        raise RuntimeError(f'File does not exist: {file}')
+
+    with file.open('rt') as fh:
+        content = json.load(fh)
+
+    copy = file.with_suffix('.json2')
+    try:
+        with copy.open('wt') as fh:
+            json.dump(content, fh, indent=indent)
+        copy.rename(file)
+    finally:
+        if copy.is_file():
+            copy.unlink()
+
+
+def format_html_string(html: str) -> str:
+    soup = BeautifulSoup(html)
+    return soup.prettify()
+
+
+def format_html_file(file: pathlib.Path):
+    """Reformat the contents of an HTML file to include indentation to increase readability."""
+    if not file or not file.is_file():
+        raise RuntimeError(f'File does not exist: {file}')
+
+    pretty = format_html_string(file.read_text())
+
+    copy = file.with_suffix('.html2')
+    try:
+        with copy.open('wt') as fh:
+            fh.write(pretty)
+        copy.rename(file)
+    finally:
+        if copy.is_file():
+            copy.unlink()

wrolpi/files/test/test_lib.py

@@ -183,7 +183,7 @@ async def test__upsert_files(test_session, make_files_structure, test_directory,
     assert_file_groups([
         {'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': False,
          'files': [
-             {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
+             {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'application/x-subrip'},
              {'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
          ]},
         {'primary_path': bar, 'idempotency': idempotency, 'indexed': False,
@@ -210,7 +210,7 @@ async def test__upsert_files(test_session, make_files_structure, test_directory,
     assert_file_groups([
         {'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True,
          'files': [
-             {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'},
+             {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'application/x-subrip'},
              {'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'},
          ]},
         {'primary_path': bar, 'idempotency': idempotency, 'indexed': False,

wrolpi/test/test_common.py

@@ -1,5 +1,6 @@
 import asyncio
 import ctypes.wintypes
+import json
 import multiprocessing
 import os
 import pathlib
@@ -719,3 +720,65 @@ Paragraph outside an element.'''
 )
 def test_human_bandwidth(bps, expected):
     assert common.human_bandwidth(bps) == expected
+
+
+def test_format_json_file():
+    with tempfile.TemporaryDirectory() as d:
+        with tempfile.NamedTemporaryFile(suffix='.json', dir=d) as fh:
+            file = pathlib.Path(fh.name)
+            file.write_text(json.dumps({
+                'one': 1,
+                'two': 2,
+                'three': 3,
+                'four': 4,
+                'five': 5,
+                'six': 6,
+                'seven': 7,
+                'eight': 8,
+                'nine': 9,
+                'ten': 10,
+            }))
+            # Only one file was created.
+            assert len(list(file.parent.iterdir())) == 1
+
+            unformatted = '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10}'
+            assert file.read_text() == unformatted
+
+            common.format_json_file(file)
+            assert file.read_text() == '''{
+  "one": 1,
+  "two": 2,
+  "three": 3,
+  "four": 4,
+  "five": 5,
+  "six": 6,
+  "seven": 7,
+  "eight": 8,
+  "nine": 9,
+  "ten": 10
+}'''
+            # The copy was deleted.
+            assert len(list(file.parent.iterdir())) == 1
+
+
+def test_format_html_file():
+    with tempfile.NamedTemporaryFile() as fh:
+        file = pathlib.Path(fh.name)
+        unformatted = '<html><!--Comment--><head><title>Title</title><body>Body</body></html>'
+        file.write_text(unformatted)
+
+        assert file.read_text() == unformatted
+
+        common.format_html_file(file)
+        assert file.read_text() == '''<html>
+ <!--Comment-->
+ <head>
+  <title>
+   Title
+  </title>
+ </head>
+ <body>
+  Body
+ </body>
+</html>
+'''