mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
Sorting keys when saving json file.
This commit is contained in:
parent
305764ee38
commit
0bee458b79
7 changed files with 70 additions and 82 deletions
|
|
@ -163,13 +163,13 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
|
|||
|
||||
if readability:
|
||||
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
|
||||
with archive_files.readability.open('wt') as fh:
|
||||
with archive_files.readability.open('wt') as fp:
|
||||
content = format_html_string(readability.pop('content'))
|
||||
fh.write(content)
|
||||
with archive_files.readability_txt.open('wt') as fh:
|
||||
fp.write(content)
|
||||
with archive_files.readability_txt.open('wt') as fp:
|
||||
readability_txt = readability.pop('textContent')
|
||||
readability_txt = split_lines_by_length(readability_txt)
|
||||
fh.write(readability_txt)
|
||||
fp.write(readability_txt)
|
||||
else:
|
||||
# No readability was returned, so there are no files.
|
||||
archive_files.readability_txt = archive_files.readability = None
|
||||
|
|
@ -186,8 +186,8 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
|
|||
# Always write a JSON file that contains at least the URL.
|
||||
readability = readability or {}
|
||||
readability['url'] = url
|
||||
with archive_files.readability_json.open('wt') as fh:
|
||||
fh.write(json.dumps(readability, indent=2))
|
||||
with archive_files.readability_json.open('wt') as fp:
|
||||
json.dump(readability, fp, indent=2, sort_keys=True)
|
||||
|
||||
# Create any File models that we can, index them all.
|
||||
paths = (
|
||||
|
|
|
|||
|
|
@ -584,7 +584,7 @@ def update_channel_catalog(channel: Channel, info: dict):
|
|||
if channel.directory:
|
||||
info_json_path = channel.directory / f'{channel.name}.info.json'
|
||||
with info_json_path.open('wt') as fh:
|
||||
json.dump(info, fh, indent=2)
|
||||
json.dump(info, fh, indent=2, sort_keys=True)
|
||||
logger.debug(f'Wrote channel info json to {info_json_path}')
|
||||
else:
|
||||
logger.debug(f'Skipping channel info json because it does not have a directory: {channel}')
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ from sqlalchemy.orm.collections import InstrumentedList
|
|||
|
||||
from modules.videos.errors import UnknownVideo, UnknownChannel
|
||||
from wrolpi.captions import read_captions
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, replace_file
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.tags import Tag, TagFile
|
||||
|
|
|
|||
|
|
@ -213,6 +213,7 @@ __all__ = [
|
|||
'register_modeler',
|
||||
'register_refresh_cleanup',
|
||||
'remove_whitespace',
|
||||
'replace_file',
|
||||
'resolve_generators',
|
||||
'run_after',
|
||||
'set_global_log_level',
|
||||
|
|
@ -1694,6 +1695,48 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
|
|||
return headlines
|
||||
|
||||
|
||||
def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False):
|
||||
"""Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful.
|
||||
The original file will be restored if any of these steps fail."""
|
||||
file = pathlib.Path(file)
|
||||
|
||||
if missing_ok is False and not file.is_file():
|
||||
raise FileNotFoundError(f'Cannot replace non-existent file: {str(file)}')
|
||||
|
||||
temporary_path = pathlib.Path(str(file) + '.tmp')
|
||||
if temporary_path.exists():
|
||||
raise RuntimeError(f'Cannot replace file, temporary path already exists! {str(temporary_path)}')
|
||||
|
||||
if not contents:
|
||||
raise RuntimeError('Refusing to replace file with empty contents')
|
||||
|
||||
try:
|
||||
if file.exists():
|
||||
file.rename(temporary_path)
|
||||
mode = 'wb' if isinstance(contents, bytes) else 'wt'
|
||||
with file.open(mode) as fp:
|
||||
fp.write(contents)
|
||||
os.fsync(fp.fileno())
|
||||
if file.exists() and temporary_path.exists():
|
||||
# New contents have been written, delete the old file.
|
||||
temporary_path.unlink()
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to replace file: {str(file)}', exc_info=e)
|
||||
if temporary_path.exists():
|
||||
# Move original file back.
|
||||
if file.exists():
|
||||
file.unlink()
|
||||
temporary_path.rename(file)
|
||||
elif file.exists():
|
||||
logger.error(f'Replacing file, but original file still exists: {str(file)}')
|
||||
else:
|
||||
logger.critical('Neither original, nor temporary file exist. I am so sorry.')
|
||||
raise
|
||||
finally:
|
||||
if file.is_file() and file.stat().st_size > 0 and temporary_path.is_file():
|
||||
temporary_path.unlink()
|
||||
|
||||
|
||||
def format_json_file(file: pathlib.Path, indent: int = 2):
|
||||
"""Reformat the contents of a JSON file to include indentation to increase readability."""
|
||||
if not indent or indent <= 0:
|
||||
|
|
@ -1701,17 +1744,9 @@ def format_json_file(file: pathlib.Path, indent: int = 2):
|
|||
if not file or not file.is_file():
|
||||
raise RuntimeError(f'File does not exist: {file}')
|
||||
|
||||
with file.open('rt') as fh:
|
||||
content = json.load(fh)
|
||||
|
||||
copy = file.with_suffix('.json2')
|
||||
try:
|
||||
with copy.open('wt') as fh:
|
||||
json.dump(content, fh, indent=indent)
|
||||
copy.rename(file)
|
||||
finally:
|
||||
if copy.is_file():
|
||||
copy.unlink()
|
||||
content = json.loads(file.read_text())
|
||||
content = json.dumps(content, indent=indent, sort_keys=True)
|
||||
replace_file(file, content)
|
||||
|
||||
|
||||
def format_html_string(html: str) -> str:
|
||||
|
|
@ -1728,15 +1763,7 @@ def format_html_file(file: pathlib.Path):
|
|||
raise RuntimeError(f'File does not exist: {file}')
|
||||
|
||||
pretty = format_html_string(file.read_text())
|
||||
|
||||
copy = file.with_suffix('.html2')
|
||||
try:
|
||||
with copy.open('wt') as fh:
|
||||
fh.write(pretty)
|
||||
copy.rename(file)
|
||||
finally:
|
||||
if copy.is_file():
|
||||
copy.unlink()
|
||||
replace_file(file, pretty)
|
||||
|
||||
|
||||
def html_screenshot(html: bytes) -> bytes:
|
||||
|
|
|
|||
|
|
@ -1298,47 +1298,6 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
|
|||
return new_path
|
||||
|
||||
|
||||
def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False):
|
||||
"""Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful.
|
||||
The original file will be restored if any of these steps fail."""
|
||||
file = pathlib.Path(file)
|
||||
|
||||
if missing_ok is False and not file.is_file():
|
||||
raise FileNotFoundError(f'Cannot replace non-existent file: {str(file)}')
|
||||
|
||||
temporary_path = pathlib.Path(str(file) + '.tmp')
|
||||
if temporary_path.exists():
|
||||
raise RuntimeError(f'Cannot replace file, temporary path already exists! {str(temporary_path)}')
|
||||
|
||||
if not contents:
|
||||
raise RuntimeError('Refusing to replace file with empty contents')
|
||||
|
||||
try:
|
||||
if file.exists():
|
||||
file.rename(temporary_path)
|
||||
mode = 'wb' if isinstance(contents, bytes) else 'wt'
|
||||
with file.open(mode) as fp:
|
||||
fp.write(contents)
|
||||
if file.exists() and temporary_path.exists():
|
||||
# New contents have been written, delete the old file.
|
||||
temporary_path.unlink()
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to replace file: {str(file)}', exc_info=e)
|
||||
if temporary_path.exists():
|
||||
# Move original file back.
|
||||
if file.exists():
|
||||
file.unlink()
|
||||
temporary_path.rename(file)
|
||||
elif file.exists():
|
||||
logger.error(f'Replacing file, but original file still exists: {str(file)}')
|
||||
else:
|
||||
logger.critical('Neither original, nor temporary file exist. I am so sorry.')
|
||||
raise
|
||||
finally:
|
||||
if file.is_file() and file.stat().st_size > 0 and temporary_path.is_file():
|
||||
temporary_path.unlink()
|
||||
|
||||
|
||||
async def rename_directory(directory: pathlib.Path, new_name: str) -> pathlib.Path:
|
||||
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
|
||||
new_directory = directory.with_name(new_name)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import mock
|
|||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
import wrolpi.common
|
||||
from wrolpi.common import timer
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
|
||||
|
|
@ -1058,17 +1059,17 @@ def test_replace_file(test_directory):
|
|||
file.write_text('old contents')
|
||||
assert file.read_text() == 'old contents'
|
||||
|
||||
lib.replace_file(file, 'new contents')
|
||||
wrolpi.common.replace_file(file, 'new contents')
|
||||
assert file.read_text() == 'new contents'
|
||||
assert not (test_directory / 'foo.txt.tmp').exists()
|
||||
|
||||
lib.replace_file(file, b'new bytes')
|
||||
wrolpi.common.replace_file(file, b'new bytes')
|
||||
assert file.read_bytes() == b'new bytes'
|
||||
assert not (test_directory / 'foo.txt.tmp').exists()
|
||||
|
||||
# Default behavior is to refuse to replace a non-existent file.
|
||||
with pytest.raises(FileNotFoundError):
|
||||
lib.replace_file('does not exist', 'foo')
|
||||
wrolpi.common.replace_file('does not exist', 'foo')
|
||||
|
||||
# Non-existent file can be created.
|
||||
lib.replace_file(test_directory / 'will now exist', 'foo', missing_ok=True)
|
||||
wrolpi.common.replace_file(test_directory / 'will now exist', 'foo', missing_ok=True)
|
||||
|
|
|
|||
|
|
@ -726,17 +726,18 @@ def test_format_json_file():
|
|||
assert file.read_text() == unformatted
|
||||
|
||||
common.format_json_file(file)
|
||||
# Keys are sorted.
|
||||
assert file.read_text() == '''{
|
||||
"one": 1,
|
||||
"two": 2,
|
||||
"three": 3,
|
||||
"four": 4,
|
||||
"five": 5,
|
||||
"six": 6,
|
||||
"seven": 7,
|
||||
"eight": 8,
|
||||
"five": 5,
|
||||
"four": 4,
|
||||
"nine": 9,
|
||||
"ten": 10
|
||||
"one": 1,
|
||||
"seven": 7,
|
||||
"six": 6,
|
||||
"ten": 10,
|
||||
"three": 3,
|
||||
"two": 2
|
||||
}'''
|
||||
# The copy was deleted.
|
||||
assert len(list(file.parent.iterdir())) == 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue