From 0bee458b7970a391b31e82ab38ce549f8bd9603d Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 13 Jun 2024 14:08:07 -0600 Subject: [PATCH] Sorting keys when saving json file. --- modules/archive/lib.py | 12 +++---- modules/videos/downloader.py | 2 +- modules/videos/models.py | 4 +-- wrolpi/common.py | 67 ++++++++++++++++++++++++----------- wrolpi/files/lib.py | 41 --------------------- wrolpi/files/test/test_lib.py | 9 ++--- wrolpi/test/test_common.py | 17 ++++----- 7 files changed, 70 insertions(+), 82 deletions(-) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 31e697d2..9c765b34 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -163,13 +163,13 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr if readability: # Write the readability parts to their own files. Write what is left after pops to the JSON file. - with archive_files.readability.open('wt') as fh: + with archive_files.readability.open('wt') as fp: content = format_html_string(readability.pop('content')) - fh.write(content) - with archive_files.readability_txt.open('wt') as fh: + fp.write(content) + with archive_files.readability_txt.open('wt') as fp: readability_txt = readability.pop('textContent') readability_txt = split_lines_by_length(readability_txt) - fh.write(readability_txt) + fp.write(readability_txt) else: # No readability was returned, so there are no files. archive_files.readability_txt = archive_files.readability = None @@ -186,8 +186,8 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr # Always write a JSON file that contains at least the URL. readability = readability or {} readability['url'] = url - with archive_files.readability_json.open('wt') as fh: - fh.write(json.dumps(readability, indent=2)) + with archive_files.readability_json.open('wt') as fp: + json.dump(readability, fp, indent=2, sort_keys=True) # Create any File models that we can, index them all. paths = ( diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 79a83cd2..d4b212e4 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -584,7 +584,7 @@ def update_channel_catalog(channel: Channel, info: dict): if channel.directory: info_json_path = channel.directory / f'{channel.name}.info.json' with info_json_path.open('wt') as fh: - json.dump(info, fh, indent=2) + json.dump(info, fh, indent=2, sort_keys=True) logger.debug(f'Wrote channel info json to {info_json_path}') else: logger.debug(f'Skipping channel info json because it does not have a directory: {channel}') diff --git a/modules/videos/models.py b/modules/videos/models.py index 922c99e5..246c442b 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -9,10 +9,10 @@ from sqlalchemy.orm.collections import InstrumentedList from modules.videos.errors import UnknownVideo, UnknownChannel from wrolpi.captions import read_captions -from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task +from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.downloader import Download, download_manager -from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, replace_file +from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix from wrolpi.files.models import FileGroup from wrolpi.media_path import MediaPathType from wrolpi.tags import Tag, TagFile diff --git a/wrolpi/common.py b/wrolpi/common.py index e11e1a73..4645c923 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -213,6 +213,7 @@ __all__ = [ 'register_modeler', 'register_refresh_cleanup', 'remove_whitespace', + 'replace_file', 'resolve_generators', 'run_after', 'set_global_log_level', @@ -1694,6 +1695,48 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl return headlines +def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False): + """Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful. + The original file will be restored if any of these steps fail.""" + file = pathlib.Path(file) + + if missing_ok is False and not file.is_file(): + raise FileNotFoundError(f'Cannot replace non-existent file: {str(file)}') + + temporary_path = pathlib.Path(str(file) + '.tmp') + if temporary_path.exists(): + raise RuntimeError(f'Cannot replace file, temporary path already exists! {str(temporary_path)}') + + if not contents: + raise RuntimeError('Refusing to replace file with empty contents') + + try: + if file.exists(): + file.rename(temporary_path) + mode = 'wb' if isinstance(contents, bytes) else 'wt' + with file.open(mode) as fp: + fp.write(contents) + os.fsync(fp.fileno()) + if file.exists() and temporary_path.exists(): + # New contents have been written, delete the old file. + temporary_path.unlink() + except Exception as e: + logger.error(f'Failed to replace file: {str(file)}', exc_info=e) + if temporary_path.exists(): + # Move original file back. + if file.exists(): + file.unlink() + temporary_path.rename(file) + elif file.exists(): + logger.error(f'Replacing file, but original file still exists: {str(file)}') + else: + logger.critical('Neither original, nor temporary file exist. I am so sorry.') + raise + finally: + if file.is_file() and file.stat().st_size > 0 and temporary_path.is_file(): + temporary_path.unlink() + + def format_json_file(file: pathlib.Path, indent: int = 2): """Reformat the contents of a JSON file to include indentation to increase readability.""" if not indent or indent <= 0: @@ -1701,17 +1744,9 @@ def format_json_file(file: pathlib.Path, indent: int = 2): if not file or not file.is_file(): raise RuntimeError(f'File does not exist: {file}') - with file.open('rt') as fh: - content = json.load(fh) - - copy = file.with_suffix('.json2') - try: - with copy.open('wt') as fh: - json.dump(content, fh, indent=indent) - copy.rename(file) - finally: - if copy.is_file(): - copy.unlink() + content = json.loads(file.read_text()) + content = json.dumps(content, indent=indent, sort_keys=True) + replace_file(file, content) def format_html_string(html: str) -> str: @@ -1728,15 +1763,7 @@ def format_html_file(file: pathlib.Path): raise RuntimeError(f'File does not exist: {file}') pretty = format_html_string(file.read_text()) - - copy = file.with_suffix('.html2') - try: - with copy.open('wt') as fh: - fh.write(pretty) - copy.rename(file) - finally: - if copy.is_file(): - copy.unlink() + replace_file(file, pretty) def html_screenshot(html: bytes) -> bytes: diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index f9e96482..e20eef62 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1298,47 +1298,6 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path: return new_path -def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False): - """Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful. - The original file will be restored if any of these steps fail.""" - file = pathlib.Path(file) - - if missing_ok is False and not file.is_file(): - raise FileNotFoundError(f'Cannot replace non-existent file: {str(file)}') - - temporary_path = pathlib.Path(str(file) + '.tmp') - if temporary_path.exists(): - raise RuntimeError(f'Cannot replace file, temporary path already exists! {str(temporary_path)}') - - if not contents: - raise RuntimeError('Refusing to replace file with empty contents') - - try: - if file.exists(): - file.rename(temporary_path) - mode = 'wb' if isinstance(contents, bytes) else 'wt' - with file.open(mode) as fp: - fp.write(contents) - if file.exists() and temporary_path.exists(): - # New contents have been written, delete the old file. - temporary_path.unlink() - except Exception as e: - logger.error(f'Failed to replace file: {str(file)}', exc_info=e) - if temporary_path.exists(): - # Move original file back. - if file.exists(): - file.unlink() - temporary_path.rename(file) - elif file.exists(): - logger.error(f'Replacing file, but original file still exists: {str(file)}') - else: - logger.critical('Neither original, nor temporary file exist. I am so sorry.') - raise - finally: - if file.is_file() and file.stat().st_size > 0 and temporary_path.is_file(): - temporary_path.unlink() - - async def rename_directory(directory: pathlib.Path, new_name: str) -> pathlib.Path: """Rename a directory. This is done by moving all files into the new directory, and removing the old directory.""" new_directory = directory.with_name(new_name) diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 27301c02..678313ee 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -12,6 +12,7 @@ import mock import pytest from PIL import Image +import wrolpi.common from wrolpi.common import timer from wrolpi.dates import now from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile @@ -1058,17 +1059,17 @@ def test_replace_file(test_directory): file.write_text('old contents') assert file.read_text() == 'old contents' - lib.replace_file(file, 'new contents') + wrolpi.common.replace_file(file, 'new contents') assert file.read_text() == 'new contents' assert not (test_directory / 'foo.txt.tmp').exists() - lib.replace_file(file, b'new bytes') + wrolpi.common.replace_file(file, b'new bytes') assert file.read_bytes() == b'new bytes' assert not (test_directory / 'foo.txt.tmp').exists() # Default behavior is to refuse to replace a non-existent file. with pytest.raises(FileNotFoundError): - lib.replace_file('does not exist', 'foo') + wrolpi.common.replace_file('does not exist', 'foo') # Non-existent file can be created. - lib.replace_file(test_directory / 'will now exist', 'foo', missing_ok=True) + wrolpi.common.replace_file(test_directory / 'will now exist', 'foo', missing_ok=True) diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 4ffc0c05..5355df7e 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -726,17 +726,18 @@ def test_format_json_file(): assert file.read_text() == unformatted common.format_json_file(file) + # Keys are sorted. assert file.read_text() == '''{ - "one": 1, - "two": 2, - "three": 3, - "four": 4, - "five": 5, - "six": 6, - "seven": 7, "eight": 8, + "five": 5, + "four": 4, "nine": 9, - "ten": 10 + "one": 1, + "seven": 7, + "six": 6, + "ten": 10, + "three": 3, + "two": 2 }''' # The copy was deleted. assert len(list(file.parent.iterdir())) == 1