From 2b811d1a51da4471c4ce4063f2d41b81b26f6b6b Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 10 Jan 2025 12:12:39 -0700 Subject: [PATCH] Removing extra meta4 files after aria2c download. --- docker/api/Dockerfile | 2 +- wrolpi/downloader.py | 15 ++++++++++++++- wrolpi/files/lib.py | 7 +++---- wrolpi/files/test/test_lib.py | 6 +++--- wrolpi/status.py | 2 +- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index 960a9cb3..1b267f74 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /opt/wrolpi # Update dependencies for the services install. RUN apt update # ffmpeg for Videos -RUN apt-get install -y ffmpeg catdoc sysstat +RUN apt-get install -y ffmpeg catdoc sysstat aria2 RUN ffmpeg -version # yt-dlp needs cache directory RUN mkdir /.cache && chown -R 1000:1000 /.cache diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 8db4dce9..ab70a957 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -30,7 +30,7 @@ from wrolpi.api_utils import api_app, perpetual_signal from wrolpi.cmd import which from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \ wrol_mode_enabled, background_task, get_wrolpi_config, get_absolute_media_path, timer, aiohttp_get, \ - get_download_info, trim_file_name + get_download_info, trim_file_name, TRACE_LEVEL from wrolpi.dates import TZDateTime, now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError, DownloadError @@ -404,6 +404,11 @@ class Downloader: async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True) \ -> pathlib.Path: + from wrolpi.files.lib import glob_shared_stem + + if not ARIA2C_PATH: + raise DownloadError('Cannot find aria2c executable') + meta4_contents = None if check_for_meta4: meta4_contents = await self.get_meta4_contents(url) @@ -445,6 +450,14 @@ class Downloader: if not output_path.is_file(): raise DownloadError(f'{error}\n\nOutput file not found: {output_path}') + # Delete any trailing meta4 files. + matching_files = glob_shared_stem(output_path) + for file in matching_files: + if file.name.endswith('.meta4'): + if logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'Deleting meta4 file: {file}') + file.unlink() + return output_path diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 69d0d519..f4a8b6a2 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -888,13 +888,12 @@ def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]: >>> glob_shared_stem(pathlib.Path('foo.mp4')) ['foo.mp4', 'foo.png', 'foo.info.json'] """ - if isinstance(path, str): - path = pathlib.Path(path) + path = path if isinstance(path, pathlib.Path) else pathlib.Path(path) stem, suffix = split_path_stem_and_suffix(path) escaped_stem = glob.escape(stem) - paths = [pathlib.Path(i) for i in path.parent.glob(f'{escaped_stem}*') if - split_path_stem_and_suffix(i)[0] == stem] + paths = [pathlib.Path(i) for i in path.parent.glob(f'{escaped_stem}*') + if i == path or i.name.startswith(f'{stem}.')] return paths diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 33617a45..b81fb5cd 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -569,9 +569,9 @@ def test_glob_shared_stem(make_files_structure): def check(path, expected): assert sorted([i.name for i in lib.glob_shared_stem(path)]) == sorted(expected) - check(mp4, ['video.mp4', 'video.png', 'video.info.json', 'video']) - check(png, ['video.mp4', 'video.png', 'video.info.json', 'video']) - check(j, ['video.mp4', 'video.png', 'video.info.json', 'video']) + check(mp4, ['video.mp4', 'video.png', 'video.info.json']) + check(png, ['video.mp4', 'video.png', 'video.info.json']) + check(j, ['video.mp4', 'video.png', 'video.info.json']) check(video, ['video.mp4', 'video.png', 'video.info.json', 'video']) check(something, ['something']) diff --git a/wrolpi/status.py b/wrolpi/status.py index e4ee7efb..ad90d9a1 100755 --- a/wrolpi/status.py +++ b/wrolpi/status.py @@ -98,7 +98,7 @@ async def get_iostat_stats() -> IostatInfo: if iostat_stats: cpu_stats, *_ = iostat_stats - logger.debug( + logger.trace( f'get_iostat_stats got percent_iowait={cpu_stats["percent_iowait"]} percent_idle={cpu_stats["percent_idle"]}') # Return only the CPU stats. return IostatInfo(