mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
HTML and JSON files are reformatted when downloaded.
This commit is contained in:
parent
17a1aa903c
commit
6959c7a3b4
8 changed files with 152 additions and 17 deletions
|
|
@ -16,7 +16,8 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from modules.archive.models import Domain, Archive
|
||||
from wrolpi.cmd import READABILITY_BIN, SINGLE_FILE_BIN, CHROMIUM
|
||||
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post
|
||||
from wrolpi.common import get_media_directory, logger, extract_domain, chdir, escape_file_name, aiohttp_post, \
|
||||
format_html_string, split_lines_by_length
|
||||
from wrolpi.dates import now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy
|
||||
|
|
@ -238,15 +239,18 @@ async def do_archive(url: str) -> Archive:
|
|||
if readability:
|
||||
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
|
||||
with archive_files.readability.open('wt') as fh:
|
||||
fh.write(readability.pop('content'))
|
||||
content = format_html_string(readability.pop('content'))
|
||||
fh.write(content)
|
||||
with archive_files.readability_txt.open('wt') as fh:
|
||||
readability_txt = readability.pop('textContent')
|
||||
readability_txt = split_lines_by_length(readability_txt)
|
||||
fh.write(readability_txt)
|
||||
else:
|
||||
# No readability was returned, so there are no files.
|
||||
archive_files.readability_txt = archive_files.readability = None
|
||||
|
||||
# Store the single-file HTML in its own file.
|
||||
singlefile = format_html_string(singlefile)
|
||||
archive_files.singlefile.write_text(singlefile)
|
||||
|
||||
if screenshot:
|
||||
|
|
@ -258,7 +262,7 @@ async def do_archive(url: str) -> Archive:
|
|||
readability = readability or {}
|
||||
readability['url'] = url
|
||||
with archive_files.readability_json.open('wt') as fh:
|
||||
fh.write(json.dumps(readability))
|
||||
fh.write(json.dumps(readability, indent=2))
|
||||
|
||||
# Create any File models that we can, index them all.
|
||||
paths = (
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ def make_fake_request_archive(readability=True, screenshot=True, title=True):
|
|||
Image.new('RGB', (1, 1), color='grey').save(fh)
|
||||
fh.seek(0)
|
||||
image = fh.read()
|
||||
singlefile = '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
singlefile = '<html><!--\n Page saved with SingleFile-->\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
r = dict(
|
||||
content=f'<html>test readability content</html>',
|
||||
textContent='test readability textContent',
|
||||
|
|
@ -298,15 +298,35 @@ async def test_new_archive(test_session, fake_now):
|
|||
assert archive1.url is not None
|
||||
assert archive1.domain is not None
|
||||
|
||||
# The actual files were dumped and read correctly.
|
||||
with open(archive1.singlefile_path) as fh:
|
||||
assert fh.read() == '<html><!--\n Page saved with SingleFile\ntest single-file\nジにてこちら\n<title>some title</title></html>'
|
||||
with open(archive1.readability_path) as fh:
|
||||
assert fh.read() == '<html>test readability content</html>'
|
||||
# The actual files were dumped and read correctly. The HTML/JSON files are reformatted.
|
||||
assert archive1.singlefile_path.read_text() == '''<html>
|
||||
<!--
|
||||
Page saved with SingleFile-->
|
||||
<body>
|
||||
<p>
|
||||
test single-file
|
||||
ジにてこちら
|
||||
</p>
|
||||
<title>
|
||||
some title
|
||||
</title>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
assert archive1.readability_path.read_text() == '''<html>
|
||||
<body>
|
||||
<p>
|
||||
test readability content
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
with open(archive1.readability_txt_path) as fh:
|
||||
assert fh.read() == 'test readability textContent'
|
||||
with open(archive1.readability_json_path) as fh:
|
||||
assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'}
|
||||
assert archive1.readability_json_path.read_text() == '''{
|
||||
"title": "\\u30b8\\u306b\\u3066\\u3053\\u3061\\u3089",
|
||||
"url": "https://example.com"
|
||||
}'''
|
||||
|
||||
fake_now(datetime(2000, 1, 2))
|
||||
archive2 = await do_archive('https://example.com')
|
||||
|
|
@ -613,7 +633,9 @@ async def test_archive_download_index(test_session, test_directory, image_file):
|
|||
|
||||
assert isinstance(archive, lib.Archive), 'Did not get an archive'
|
||||
assert archive.singlefile_path and archive.singlefile_path.exists() and \
|
||||
archive.singlefile_path.read_text() == singlefile, 'Singlefile was not stored'
|
||||
archive.singlefile_path.read_text() == \
|
||||
'<html>\n <head>\n <title>\n the singlefile\n </title>\n </head>\n</html>\n', \
|
||||
'Singlefile was not formatted'
|
||||
assert archive.readability_path and archive.readability_path.exists()
|
||||
assert archive.readability_json_path
|
||||
assert archive.readability_txt_path and archive.readability_txt_path.exists()
|
||||
|
|
|
|||
|
|
@ -13,7 +13,8 @@ from yt_dlp import YoutubeDL
|
|||
from yt_dlp.extractor import YoutubeTabIE # noqa
|
||||
|
||||
from wrolpi.cmd import YT_DLP_BIN
|
||||
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task
|
||||
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
|
||||
format_json_file
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.db import optional_session
|
||||
|
|
@ -331,6 +332,9 @@ class VideoDownloader(Downloader, ABC):
|
|||
video.channel_id = channel_id
|
||||
video_id = video.id
|
||||
|
||||
if video.info_json_path:
|
||||
format_json_file(video.info_json_path)
|
||||
|
||||
session.commit()
|
||||
|
||||
if download.settings and (tag_names := download.settings.get('tag_names')):
|
||||
|
|
|
|||
|
|
@ -276,7 +276,8 @@ class Video(ModelHelper, Base):
|
|||
|
||||
@property
|
||||
def caption_files(self) -> List[dict]:
|
||||
return self.file_group.my_files('text/vtt') + self.file_group.my_files('text/srt')
|
||||
return self.file_group.my_files('text/vtt') + self.file_group.my_files('text/srt') \
|
||||
+ self.file_group.my_files('application/x-subrip')
|
||||
|
||||
@property
|
||||
def caption_paths(self) -> List[pathlib.Path]:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue