HTML and JSON files are reformatted when downloaded.

This commit is contained in:
Roland 2023-08-26 21:47:53 -06:00
parent 17a1aa903c
commit 6959c7a3b4
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
8 changed files with 152 additions and 17 deletions

View file

@ -16,7 +16,8 @@ from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post, \
format_html_string, split_lines_by_length
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy
@ -238,15 +239,18 @@ async def do_archive(url: str) -> Archive:
if readability:
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
with archive_files.readability.open('wt') as fh:
fh.write(readability.pop('content'))
content = format_html_string(readability.pop('content'))
fh.write(content)
with archive_files.readability_txt.open('wt') as fh:
readability_txt = readability.pop('textContent')
readability_txt = split_lines_by_length(readability_txt)
fh.write(readability_txt)
else:
# No readability was returned, so there are no files.
archive_files.readability_txt = archive_files.readability = None
# Store the single-file HTML in its own file.
singlefile = format_html_string(singlefile)
archive_files.singlefile.write_text(singlefile)
if screenshot:
@ -258,7 +262,7 @@ async def do_archive(url: str) -> Archive:
readability = readability or {}
readability['url'] = url
with archive_files.readability_json.open('wt') as fh:
fh.write(json.dumps(readability))
fh.write(json.dumps(readability, indent=2))
# Create any File models that we can, index them all.
paths = (

View file

@ -24,7 +24,7 @@ def make_fake_request_archive(readability=True, screenshot=True, title=True):
Image.new('RGB', (1, 1), color='grey').save(fh)
fh.seek(0)
image = fh.read()
singlefile = '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
singlefile = '<html><!--\n Page saved with SingleFile-->\ntest single-file\nジにてこちら\n<title>some title</title></html>'
r = dict(
content=f'<html>test readability content</html>',
textContent='test readability textContent',
@ -298,15 +298,35 @@ async def test_new_archive(test_session, fake_now):
assert archive1.url is not None
assert archive1.domain is not None
# The actual files were dumped and read correctly.
with open(archive1.singlefile_path) as fh:
assert fh.read() == '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
with open(archive1.readability_path) as fh:
assert fh.read() == '<html>test readability content</html>'
# The actual files were dumped and read correctly. The HTML/JSON files are reformatted.
assert archive1.singlefile_path.read_text() == '''<html>
<!--
Page saved with SingleFile-->
<body>
<p>
test single-file
ジにてこちら
</p>
<title>
some title
</title>
</body>
</html>
'''
assert archive1.readability_path.read_text() == '''<html>
<body>
<p>
test readability content
</p>
</body>
</html>
'''
with open(archive1.readability_txt_path) as fh:
assert fh.read() == 'test readability textContent'
with open(archive1.readability_json_path) as fh:
assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'}
assert archive1.readability_json_path.read_text() == '''{
"title": "\\u30b8\\u306b\\u3066\\u3053\\u3061\\u3089",
"url": "https://example.com"
}'''
fake_now(datetime(2000, 1, 2))
archive2 = await do_archive('https://example.com')
@ -613,7 +633,9 @@ async def test_archive_download_index(test_session, test_directory, image_file):
assert isinstance(archive, lib.Archive), 'Did not get an archive'
assert archive.singlefile_path and archive.singlefile_path.exists() and \
archive.singlefile_path.read_text() == singlefile, 'Singlefile was not stored'
archive.singlefile_path.read_text() == \
'<html>\n <head>\n <title>\n the singlefile\n </title>\n </head>\n</html>\n', \
'Singlefile was not formatted'
assert archive.readability_path and archive.readability_path.exists()
assert archive.readability_json_path
assert archive.readability_txt_path and archive.readability_txt_path.exists()

View file

@ -13,7 +13,8 @@ from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE # noqa
from wrolpi.cmd import YT_DLP_BIN
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
format_json_file
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.db import optional_session
@ -331,6 +332,9 @@ class VideoDownloader(Downloader, ABC):
video.channel_id = channel_id
video_id = video.id
if video.info_json_path:
format_json_file(video.info_json_path)
session.commit()
if download.settings and (tag_names := download.settings.get('tag_names')):

View file

@ -276,7 +276,8 @@ class Video(ModelHelper, Base):
@property
def caption_files(self) -> List[dict]:
return self.file_group.my_files('text/vtt') + self.file_group.my_files('text/srt')
return self.file_group.my_files('text/vtt') + self.file_group.my_files('text/srt') \
+ self.file_group.my_files('application/x-subrip')
@property
def caption_paths(self) -> List[pathlib.Path]: