Trimming video file names when downloading. Using yt-dlp error to get filename to help yt-dlp extract info json.

This commit is contained in:
Roland 2024-10-03 12:16:44 -06:00
parent 05b206813b
commit b3380aeacb
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
7 changed files with 72 additions and 44 deletions

View file

@ -8,7 +8,7 @@ from typing import Tuple, List, Dict, Optional
import yt_dlp.utils
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from yt_dlp import YoutubeDL, DownloadError
from yt_dlp.extractor import YoutubeTabIE # noqa
from wrolpi.cmd import YT_DLP_BIN
@ -19,7 +19,7 @@ from wrolpi.db import get_db_session
from wrolpi.db import optional_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.lib import glob_shared_stem
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
from .channel.lib import create_channel, get_channel
@ -503,8 +503,30 @@ class VideoDownloader(Downloader, ABC):
ydl.add_default_info_extractors()
# Get the path where the video will be saved.
entry = extract_info(url, ydl=ydl, process=True)
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
try:
entry = extract_info(url, ydl=ydl, process=True)
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
final_filename = trim_file_name(final_filename)
except DownloadError as e:
if ' Cannot write ' in str(e):
# yt-dlp does not handle long file names well, get the name from the error (lol)
last_line = str(e).splitlines()[-1]
filename = last_line.split(' file ')[-1].strip()
full_path, _ = split_path_stem_and_suffix(filename, full=True)
if not full_path.startswith('/'):
logger.error(f'Failed to extract filename from {last_line}')
raise
# Trim long filename, add video suffix.
final_filename = pathlib.Path(f'{full_path}.{PREFERRED_VIDEO_EXTENSION}')
final_filename = trim_file_name(final_filename)
# Get entry info json.
options['outtmpl'] = str(final_filename)
ydl = YoutubeDL(options)
ydl.params['logger'] = ydl_logger
ydl.add_default_info_extractors()
entry = extract_info(url, ydl=ydl, process=True)
else:
raise
logger.debug(f'Downloading {url} to {repr(str(final_filename))}')
return final_filename, entry

View file

@ -12,34 +12,6 @@ if [ $EUID != 0 ]; then
exit $?
fi
Help() {
# Display Help
echo "Build WROLPi Raspberry Pi images"
echo
echo "Syntax: upgrade.sh [-h] [-b BRANCH]"
echo "options:"
echo "h Print this help."
echo "b Build from this git BRANCH."
echo
}
BRANCH="master"
while getopts ":hb:" option; do
case $option in
h) # display Help
Help
exit
;;
b)
BRANCH="${OPTARG}"
;;
*) # invalid argument(s)
echo "Error: Invalid option"
exit 1
;;
esac
done
if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/gis-map.dump.gz" ]; then
echo "stage2/04-wrolpi/files/gis-map.dump.gz does not exist!"
exit 1
@ -57,7 +29,6 @@ git clone --branch arm64 https://github.com/RPI-Distro/pi-gen.git "${BUILD_DIR}"
# Copy the configuration and build files into the pi-gen directory.
cp "${SCRIPT_DIR}/config.txt" "${BUILD_DIR}/config.txt"
echo "WROLPI_BRANCH=${BRANCH}" >> "${BUILD_DIR}/config.txt"
rsync -a "${SCRIPT_DIR}"/stage2/* "${BUILD_DIR}/stage2/"
# We only need to build the Lite and Desktop images.

View file

@ -21,8 +21,7 @@ sed -i 's/port = 5433/port = 5432/' /etc/postgresql/15/main/postgresql.conf
npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0
# Put the latest WROLPi master in /opt/wrolpi.
export WROLPI_BRANCH=${WROLPI_BRANCH:master}
git clone -b ${WROLPI_BRANCH} https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi
git clone -b master https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi
git config --global --add safe.directory /opt/wrolpi
# Install Python requirements.

View file

@ -476,7 +476,7 @@ class ConfigFile:
logger_.debug(f'Copied backup config: {rel_path} -> {backup_file}')
# Write the config in-memory to the file. Track version changes to avoid overriding newer config.
self._config['version'] += 1
self._config['version'] = (self._config['version'] or 0) + 1
config = deepcopy(self._config)
self.write_config_data(config, file)
@ -1213,22 +1213,31 @@ def escape_file_name(name: str) -> str:
# Maximum length is probably 255, but we need more length for large suffixes like `.readability.json`, and temporary
# downloading suffixes.
# downloading suffixes from yt-dlp.
MAXIMUM_FILE_LENGTH = 180
def trim_file_name(name: str) -> str:
def trim_file_name(path: str | pathlib.Path) -> str | pathlib.Path:
"""Shorten the file name only if it is longer than the file system supports. Trim from the end of the name until
the name is short enough, excluding suffix."""
if len(name) < MAXIMUM_FILE_LENGTH:
return name
the name is short enough (preserving any suffix)."""
name_type = pathlib.Path if isinstance(path, pathlib.Path) else str
parent = path.parent if isinstance(path, pathlib.Path) else '/'.join(i for i in path.split('/')[:-1])
path = path.name if isinstance(path, pathlib.Path) else path
if len(path) < MAXIMUM_FILE_LENGTH:
return name_type(path)
# Don't trim the filename to exactly 256 characters.
# This is because a FileGroup will have varying filename lengths.
from wrolpi.files.lib import split_path_stem_and_suffix
stem, suffix = split_path_stem_and_suffix(name)
stem, suffix = split_path_stem_and_suffix(path)
excess = MAXIMUM_FILE_LENGTH - len(suffix)
return stem[:excess].strip() + suffix
new_name = stem[:excess].strip() + suffix
if parent:
if name_type == pathlib.Path:
return parent / new_name
return f'{parent}/{new_name}'
return name_type(new_name)
def native_only(func: callable):

View file

@ -61,7 +61,7 @@ def strpdate(dt: str) -> datetime:
if len(c) == 13:
# d/m/Y HH:MM:SS
return datetime.strptime(dt, '%m/%d/%Y %H:%M:%S')
elif len(c) == 15:
elif len(c) in (15, 16):
# d/m/Y HH:MM:SS PM
return datetime.strptime(dt, '%m/%d/%Y %H:%M:%S %p')
elif dt.count('-') == 2 and len(dt) <= 10:

View file

@ -1014,3 +1014,29 @@ async def test_config_valid(async_client, test_wrolpi_config):
test_wrolpi_config.write_text('wrol_mode: true')
assert config.is_valid() is True
@pytest.mark.parametrize('name,expected_name', [
('foo', 'foo'),
(pathlib.Path('foo'), pathlib.Path('foo')),
(
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.txt',
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt'
),
(
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json',
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
),
(
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json',
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
),
(
pathlib.Path(
'/some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.readability.json'),
pathlib.Path(
'/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json')
),
])
def test_trim_file_name(name, expected_name):
assert common.trim_file_name(name) == expected_name

View file

@ -119,6 +119,7 @@ def test_timedelta_to_timestamp(td, expected):
('2022-02-18T11:00:03.028Z', datetime(2022, 2, 18, 11, 0, 3, 28000, tzinfo=pytz.UTC)),
# From SingleFile.
('Fri Jun 17 2022 19:24:52', datetime(2022, 6, 17, 19, 24, 52)),
('9/26/2024 11:55:13 AM', datetime(2024, 9, 26, 11, 55, 13)),
# PDFs are the Wild West...
("D:20221226113758-07'00", datetime(2022, 12, 26, 11, 37, 58, tzinfo=timezone(timedelta(days=-1, seconds=61200)))),
('D:20200205184724', datetime(2020, 2, 5, 18, 47, 24)),