mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
ae9d188391
Parsing SingleFile datetime using dates.strpdate. Adding timezone when no timezone info is available. Installing python-dateutil to assist in handling diverse datetime strings. Reworking refresh progress to use idempotency to display more accurate progress. Only logging that files have been invalidated if they used to be indexed. Clearing archives to force re-index. Ignoring 'Unknown' author from PDF metadata. Using file_title when pdf metadata title is empty.
106 lines
3.9 KiB
Python
Executable file
106 lines
3.9 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Extracts Readability, screenshot, etc. from a Singlefile."""
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
from modules.archive import lib as archive_lib
|
|
from wrolpi.cmd import READABILITY_BIN
|
|
from wrolpi.common import html_screenshot, format_html_string, logger
|
|
|
|
logger = logger.getChild('singlefile_to_archive')
|
|
|
|
|
|
def do_readability(html: bytes, url: str) -> dict:
|
|
"""Extract the readability dict from the provided HTML."""
|
|
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
|
|
fh.write(html)
|
|
|
|
cmd = (READABILITY_BIN, fh.name, url)
|
|
output = subprocess.check_output(cmd, timeout=60)
|
|
|
|
readability = json.loads(output)
|
|
return readability
|
|
|
|
|
|
def singlefile_to_archive_files(singlefile: pathlib.Path):
|
|
"""Extract Readability files and Screenshot file next to the provided Singlefile."""
|
|
contents = singlefile.read_bytes()
|
|
|
|
readability_html_path = singlefile.with_suffix('.readability.html')
|
|
readability_json_path = singlefile.with_suffix('.readability.json')
|
|
readability_text_path = singlefile.with_suffix('.readability.txt')
|
|
|
|
# Only extract Readability if any of the files are missing.
|
|
readability = dict()
|
|
if not readability_html_path.is_file() or not readability_text_path.is_file() or not readability_json_path.is_file():
|
|
url = archive_lib.get_url_from_singlefile(contents)
|
|
try:
|
|
readability = do_readability(contents, url)
|
|
except Exception as e:
|
|
logger.error(f'Failed to run readability', exc_info=e)
|
|
|
|
# The real Readability the user wants.
|
|
readability_html = readability.pop('content', None)
|
|
if readability and not readability_html_path.is_file() and readability_html:
|
|
try:
|
|
readability_html = format_html_string(readability_html)
|
|
readability_html_path.write_text(readability_html)
|
|
except Exception as e:
|
|
logger.error(f'Failed to extract readability: {singlefile}', exc_info=e)
|
|
|
|
# The text from within the Readability, used for searching.
|
|
readability_text = readability.pop('textContent', None)
|
|
if readability and not readability_text_path.is_file() and readability_text:
|
|
readability_text_path.write_text(readability_text)
|
|
|
|
# Write the JSON last now that content/textContent have been removed.
|
|
if readability and not readability_json_path.is_file():
|
|
try:
|
|
with readability_json_path.open('wt') as fh:
|
|
fh.write(json.dumps(readability, indent=2))
|
|
except Exception as e:
|
|
logger.error(f'Failed to extract Readability text: {singlefile}', exc_info=e)
|
|
|
|
# Only generate a screenshot if the file is missing.
|
|
screenshot_path = singlefile.with_suffix('.png')
|
|
if not screenshot_path.is_file():
|
|
try:
|
|
screenshot = html_screenshot(contents)
|
|
screenshot_path.write_bytes(screenshot)
|
|
except Exception as e:
|
|
logger.error(f'Failed to create screenshot: {singlefile}', exc_info=e)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('files', nargs='+')
|
|
parser.add_argument('-v', action='count')
|
|
args = parser.parse_args()
|
|
|
|
if args.v == 0:
|
|
logger.setLevel(logging.WARNING)
|
|
elif args.v == 1:
|
|
logger.setLevel(logging.INFO)
|
|
elif args.v >= 2:
|
|
logger.setLevel(logging.DEBUG)
|
|
logger.debug(f'Debug logging')
|
|
|
|
if not READABILITY_BIN:
|
|
print(f'Cannot find "readability-extractor". Install it.')
|
|
sys.exit(1)
|
|
|
|
files = [pathlib.Path(i) for i in args.files]
|
|
invalid_files = [i for i in files if not archive_lib.is_singlefile_file(i)]
|
|
if invalid_files:
|
|
for i in invalid_files:
|
|
print(f'File is not a singlefile: {i}')
|
|
sys.exit(1)
|
|
|
|
for file in files:
|
|
singlefile_to_archive_files(file)
|
|
print(f'Finished {file}')
|