diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 110e3ac7..d43f17d5 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -8,7 +8,7 @@ from typing import Tuple, List, Dict, Optional import yt_dlp.utils from sqlalchemy.orm import Session -from yt_dlp import YoutubeDL +from yt_dlp import YoutubeDL, DownloadError from yt_dlp.extractor import YoutubeTabIE # noqa from wrolpi.cmd import YT_DLP_BIN @@ -19,7 +19,7 @@ from wrolpi.db import get_db_session from wrolpi.db import optional_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError -from wrolpi.files.lib import glob_shared_stem +from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR from .channel.lib import create_channel, get_channel @@ -503,8 +503,30 @@ class VideoDownloader(Downloader, ABC): ydl.add_default_info_extractors() # Get the path where the video will be saved. - entry = extract_info(url, ydl=ydl, process=True) - final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute() + try: + entry = extract_info(url, ydl=ydl, process=True) + final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute() + final_filename = trim_file_name(final_filename) + except DownloadError as e: + if ' Cannot write ' in str(e): + # yt-dlp does not handle long file names well, get the name from the error (lol) + last_line = str(e).splitlines()[-1] + filename = last_line.split(' file ')[-1].strip() + full_path, _ = split_path_stem_and_suffix(filename, full=True) + if not full_path.startswith('/'): + logger.error(f'Failed to extract filename from {last_line}') + raise + # Trim long filename, add video suffix. + final_filename = pathlib.Path(f'{full_path}.{PREFERRED_VIDEO_EXTENSION}') + final_filename = trim_file_name(final_filename) + # Get entry info json. + options['outtmpl'] = str(final_filename) + ydl = YoutubeDL(options) + ydl.params['logger'] = ydl_logger + ydl.add_default_info_extractors() + entry = extract_info(url, ydl=ydl, process=True) + else: + raise logger.debug(f'Downloading {url} to {repr(str(final_filename))}') return final_filename, entry diff --git a/pi-gen/build.sh b/pi-gen/build.sh index 3f00e3cd..ebd5a627 100755 --- a/pi-gen/build.sh +++ b/pi-gen/build.sh @@ -12,34 +12,6 @@ if [ $EUID != 0 ]; then exit $? fi -Help() { - # Display Help - echo "Build WROLPi Raspberry Pi images" - echo - echo "Syntax: upgrade.sh [-h] [-b BRANCH]" - echo "options:" - echo "h Print this help." - echo "b Build from this git BRANCH." - echo -} - -BRANCH="master" -while getopts ":hb:" option; do - case $option in - h) # display Help - Help - exit - ;; - b) - BRANCH="${OPTARG}" - ;; - *) # invalid argument(s) - echo "Error: Invalid option" - exit 1 - ;; - esac -done - if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/gis-map.dump.gz" ]; then echo "stage2/04-wrolpi/files/gis-map.dump.gz does not exist!" exit 1 @@ -57,7 +29,6 @@ git clone --branch arm64 https://github.com/RPI-Distro/pi-gen.git "${BUILD_DIR}" # Copy the configuration and build files into the pi-gen directory. cp "${SCRIPT_DIR}/config.txt" "${BUILD_DIR}/config.txt" -echo "WROLPI_BRANCH=${BRANCH}" >> "${BUILD_DIR}/config.txt" rsync -a "${SCRIPT_DIR}"/stage2/* "${BUILD_DIR}/stage2/" # We only need to build the Lite and Desktop images. diff --git a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh index 736853ac..3f11d01a 100755 --- a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh @@ -21,8 +21,7 @@ sed -i 's/port = 5433/port = 5432/' /etc/postgresql/15/main/postgresql.conf npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0 # Put the latest WROLPi master in /opt/wrolpi. -export WROLPI_BRANCH=${WROLPI_BRANCH:master} -git clone -b ${WROLPI_BRANCH} https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi +git clone -b master https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi git config --global --add safe.directory /opt/wrolpi # Install Python requirements. diff --git a/wrolpi/common.py b/wrolpi/common.py index ea0b9b21..e8f8da8c 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -476,7 +476,7 @@ class ConfigFile: logger_.debug(f'Copied backup config: {rel_path} -> {backup_file}') # Write the config in-memory to the file. Track version changes to avoid overriding newer config. - self._config['version'] += 1 + self._config['version'] = (self._config['version'] or 0) + 1 config = deepcopy(self._config) self.write_config_data(config, file) @@ -1213,22 +1213,31 @@ def escape_file_name(name: str) -> str: # Maximum length is probably 255, but we need more length for large suffixes like `.readability.json`, and temporary -# downloading suffixes. +# downloading suffixes from yt-dlp. MAXIMUM_FILE_LENGTH = 180 -def trim_file_name(name: str) -> str: +def trim_file_name(path: str | pathlib.Path) -> str | pathlib.Path: """Shorten the file name only if it is longer than the file system supports. Trim from the end of the name until - the name is short enough, excluding suffix.""" - if len(name) < MAXIMUM_FILE_LENGTH: - return name + the name is short enough (preserving any suffix).""" + name_type = pathlib.Path if isinstance(path, pathlib.Path) else str + parent = path.parent if isinstance(path, pathlib.Path) else '/'.join(i for i in path.split('/')[:-1]) + path = path.name if isinstance(path, pathlib.Path) else path + + if len(path) < MAXIMUM_FILE_LENGTH: + return name_type(path) # Don't trim the filename to exactly 256 characters. # This is because a FileGroup will have varying filename lengths. from wrolpi.files.lib import split_path_stem_and_suffix - stem, suffix = split_path_stem_and_suffix(name) + stem, suffix = split_path_stem_and_suffix(path) excess = MAXIMUM_FILE_LENGTH - len(suffix) - return stem[:excess].strip() + suffix + new_name = stem[:excess].strip() + suffix + if parent: + if name_type == pathlib.Path: + return parent / new_name + return f'{parent}/{new_name}' + return name_type(new_name) def native_only(func: callable): diff --git a/wrolpi/dates.py b/wrolpi/dates.py index ea71d3f2..72e95403 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -61,7 +61,7 @@ def strpdate(dt: str) -> datetime: if len(c) == 13: # d/m/Y HH:MM:SS return datetime.strptime(dt, '%m/%d/%Y %H:%M:%S') - elif len(c) == 15: + elif len(c) in (15, 16): # d/m/Y HH:MM:SS PM return datetime.strptime(dt, '%m/%d/%Y %H:%M:%S %p') elif dt.count('-') == 2 and len(dt) <= 10: diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index c50ebddc..870500d4 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1014,3 +1014,29 @@ async def test_config_valid(async_client, test_wrolpi_config): test_wrolpi_config.write_text('wrol_mode: true') assert config.is_valid() is True + + +@pytest.mark.parametrize('name,expected_name', [ + ('foo', 'foo'), + (pathlib.Path('foo'), pathlib.Path('foo')), + ( + 'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.txt', + 'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt' + ), + ( + 'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json', + 'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json' + ), + ( + '/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json', + '/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json' + ), + ( + pathlib.Path( + '/some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.readability.json'), + pathlib.Path( + '/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json') + ), +]) +def test_trim_file_name(name, expected_name): + assert common.trim_file_name(name) == expected_name diff --git a/wrolpi/test/test_dates.py b/wrolpi/test/test_dates.py index cb102fe1..b0b1c962 100644 --- a/wrolpi/test/test_dates.py +++ b/wrolpi/test/test_dates.py @@ -119,6 +119,7 @@ def test_timedelta_to_timestamp(td, expected): ('2022-02-18T11:00:03.028Z', datetime(2022, 2, 18, 11, 0, 3, 28000, tzinfo=pytz.UTC)), # From SingleFile. ('Fri Jun 17 2022 19:24:52', datetime(2022, 6, 17, 19, 24, 52)), + ('9/26/2024 11:55:13 AM', datetime(2024, 9, 26, 11, 55, 13)), # PDFs are the Wild West... ("D:20221226113758-07'00", datetime(2022, 12, 26, 11, 37, 58, tzinfo=timezone(timedelta(days=-1, seconds=61200)))), ('D:20200205184724', datetime(2020, 2, 5, 18, 47, 24)),