Sorting keys when saving json file.

This commit is contained in:
Roland 2024-06-13 14:08:07 -06:00
parent 305764ee38
commit 0bee458b79
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
7 changed files with 70 additions and 82 deletions

View file

@ -163,13 +163,13 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
if readability:
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
with archive_files.readability.open('wt') as fh:
with archive_files.readability.open('wt') as fp:
content = format_html_string(readability.pop('content'))
fh.write(content)
with archive_files.readability_txt.open('wt') as fh:
fp.write(content)
with archive_files.readability_txt.open('wt') as fp:
readability_txt = readability.pop('textContent')
readability_txt = split_lines_by_length(readability_txt)
fh.write(readability_txt)
fp.write(readability_txt)
else:
# No readability was returned, so there are no files.
archive_files.readability_txt = archive_files.readability = None
@ -186,8 +186,8 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
# Always write a JSON file that contains at least the URL.
readability = readability or {}
readability['url'] = url
with archive_files.readability_json.open('wt') as fh:
fh.write(json.dumps(readability, indent=2))
with archive_files.readability_json.open('wt') as fp:
json.dump(readability, fp, indent=2, sort_keys=True)
# Create any File models that we can, index them all.
paths = (

View file

@ -584,7 +584,7 @@ def update_channel_catalog(channel: Channel, info: dict):
if channel.directory:
info_json_path = channel.directory / f'{channel.name}.info.json'
with info_json_path.open('wt') as fh:
json.dump(info, fh, indent=2)
json.dump(info, fh, indent=2, sort_keys=True)
logger.debug(f'Wrote channel info json to {info_json_path}')
else:
logger.debug(f'Skipping channel info json because it does not have a directory: {channel}')

View file

@ -9,10 +9,10 @@ from sqlalchemy.orm.collections import InstrumentedList
from modules.videos.errors import UnknownVideo, UnknownChannel
from wrolpi.captions import read_captions
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download, download_manager
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, replace_file
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
from wrolpi.tags import Tag, TagFile

View file

@ -213,6 +213,7 @@ __all__ = [
'register_modeler',
'register_refresh_cleanup',
'remove_whitespace',
'replace_file',
'resolve_generators',
'run_after',
'set_global_log_level',
@ -1694,6 +1695,48 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
return headlines
def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False):
"""Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful.
The original file will be restored if any of these steps fail."""
file = pathlib.Path(file)
if missing_ok is False and not file.is_file():
raise FileNotFoundError(f'Cannot replace non-existent file: {str(file)}')
temporary_path = pathlib.Path(str(file) + '.tmp')
if temporary_path.exists():
raise RuntimeError(f'Cannot replace file, temporary path already exists! {str(temporary_path)}')
if not contents:
raise RuntimeError('Refusing to replace file with empty contents')
try:
if file.exists():
file.rename(temporary_path)
mode = 'wb' if isinstance(contents, bytes) else 'wt'
with file.open(mode) as fp:
fp.write(contents)
os.fsync(fp.fileno())
if file.exists() and temporary_path.exists():
# New contents have been written, delete the old file.
temporary_path.unlink()
except Exception as e:
logger.error(f'Failed to replace file: {str(file)}', exc_info=e)
if temporary_path.exists():
# Move original file back.
if file.exists():
file.unlink()
temporary_path.rename(file)
elif file.exists():
logger.error(f'Replacing file, but original file still exists: {str(file)}')
else:
logger.critical('Neither original, nor temporary file exist. I am so sorry.')
raise
finally:
if file.is_file() and file.stat().st_size > 0 and temporary_path.is_file():
temporary_path.unlink()
def format_json_file(file: pathlib.Path, indent: int = 2):
"""Reformat the contents of a JSON file to include indentation to increase readability."""
if not indent or indent <= 0:
@ -1701,17 +1744,9 @@ def format_json_file(file: pathlib.Path, indent: int = 2):
if not file or not file.is_file():
raise RuntimeError(f'File does not exist: {file}')
with file.open('rt') as fh:
content = json.load(fh)
copy = file.with_suffix('.json2')
try:
with copy.open('wt') as fh:
json.dump(content, fh, indent=indent)
copy.rename(file)
finally:
if copy.is_file():
copy.unlink()
content = json.loads(file.read_text())
content = json.dumps(content, indent=indent, sort_keys=True)
replace_file(file, content)
def format_html_string(html: str) -> str:
@ -1728,15 +1763,7 @@ def format_html_file(file: pathlib.Path):
raise RuntimeError(f'File does not exist: {file}')
pretty = format_html_string(file.read_text())
copy = file.with_suffix('.html2')
try:
with copy.open('wt') as fh:
fh.write(pretty)
copy.rename(file)
finally:
if copy.is_file():
copy.unlink()
replace_file(file, pretty)
def html_screenshot(html: bytes) -> bytes:

View file

@ -1298,47 +1298,6 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
return new_path
def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False):
"""Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful.
The original file will be restored if any of these steps fail."""
file = pathlib.Path(file)
if missing_ok is False and not file.is_file():
raise FileNotFoundError(f'Cannot replace non-existent file: {str(file)}')
temporary_path = pathlib.Path(str(file) + '.tmp')
if temporary_path.exists():
raise RuntimeError(f'Cannot replace file, temporary path already exists! {str(temporary_path)}')
if not contents:
raise RuntimeError('Refusing to replace file with empty contents')
try:
if file.exists():
file.rename(temporary_path)
mode = 'wb' if isinstance(contents, bytes) else 'wt'
with file.open(mode) as fp:
fp.write(contents)
if file.exists() and temporary_path.exists():
# New contents have been written, delete the old file.
temporary_path.unlink()
except Exception as e:
logger.error(f'Failed to replace file: {str(file)}', exc_info=e)
if temporary_path.exists():
# Move original file back.
if file.exists():
file.unlink()
temporary_path.rename(file)
elif file.exists():
logger.error(f'Replacing file, but original file still exists: {str(file)}')
else:
logger.critical('Neither original, nor temporary file exist. I am so sorry.')
raise
finally:
if file.is_file() and file.stat().st_size > 0 and temporary_path.is_file():
temporary_path.unlink()
async def rename_directory(directory: pathlib.Path, new_name: str) -> pathlib.Path:
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
new_directory = directory.with_name(new_name)

View file

@ -12,6 +12,7 @@ import mock
import pytest
from PIL import Image
import wrolpi.common
from wrolpi.common import timer
from wrolpi.dates import now
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
@ -1058,17 +1059,17 @@ def test_replace_file(test_directory):
file.write_text('old contents')
assert file.read_text() == 'old contents'
lib.replace_file(file, 'new contents')
wrolpi.common.replace_file(file, 'new contents')
assert file.read_text() == 'new contents'
assert not (test_directory / 'foo.txt.tmp').exists()
lib.replace_file(file, b'new bytes')
wrolpi.common.replace_file(file, b'new bytes')
assert file.read_bytes() == b'new bytes'
assert not (test_directory / 'foo.txt.tmp').exists()
# Default behavior is to refuse to replace a non-existent file.
with pytest.raises(FileNotFoundError):
lib.replace_file('does not exist', 'foo')
wrolpi.common.replace_file('does not exist', 'foo')
# Non-existent file can be created.
lib.replace_file(test_directory / 'will now exist', 'foo', missing_ok=True)
wrolpi.common.replace_file(test_directory / 'will now exist', 'foo', missing_ok=True)

View file

@ -726,17 +726,18 @@ def test_format_json_file():
assert file.read_text() == unformatted
common.format_json_file(file)
# Keys are sorted.
assert file.read_text() == '''{
"one": 1,
"two": 2,
"three": 3,
"four": 4,
"five": 5,
"six": 6,
"seven": 7,
"eight": 8,
"five": 5,
"four": 4,
"nine": 9,
"ten": 10
"one": 1,
"seven": 7,
"six": 6,
"ten": 10,
"three": 3,
"two": 2
}'''
# The copy was deleted.
assert len(list(file.parent.iterdir())) == 1