mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Trimming video file names when downloading. Using yt-dlp error to get filename to help yt-dlp extract info json.
This commit is contained in:
parent
05b206813b
commit
b3380aeacb
7 changed files with 72 additions and 44 deletions
|
|
@ -8,7 +8,7 @@ from typing import Tuple, List, Dict, Optional
|
|||
|
||||
import yt_dlp.utils
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp import YoutubeDL
|
||||
from yt_dlp import YoutubeDL, DownloadError
|
||||
from yt_dlp.extractor import YoutubeTabIE # noqa
|
||||
|
||||
from wrolpi.cmd import YT_DLP_BIN
|
||||
|
|
@ -19,7 +19,7 @@ from wrolpi.db import get_db_session
|
|||
from wrolpi.db import optional_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
|
||||
from .channel.lib import create_channel, get_channel
|
||||
|
|
@ -503,8 +503,30 @@ class VideoDownloader(Downloader, ABC):
|
|||
ydl.add_default_info_extractors()
|
||||
|
||||
# Get the path where the video will be saved.
|
||||
entry = extract_info(url, ydl=ydl, process=True)
|
||||
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
|
||||
try:
|
||||
entry = extract_info(url, ydl=ydl, process=True)
|
||||
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
|
||||
final_filename = trim_file_name(final_filename)
|
||||
except DownloadError as e:
|
||||
if ' Cannot write ' in str(e):
|
||||
# yt-dlp does not handle long file names well, get the name from the error (lol)
|
||||
last_line = str(e).splitlines()[-1]
|
||||
filename = last_line.split(' file ')[-1].strip()
|
||||
full_path, _ = split_path_stem_and_suffix(filename, full=True)
|
||||
if not full_path.startswith('/'):
|
||||
logger.error(f'Failed to extract filename from {last_line}')
|
||||
raise
|
||||
# Trim long filename, add video suffix.
|
||||
final_filename = pathlib.Path(f'{full_path}.{PREFERRED_VIDEO_EXTENSION}')
|
||||
final_filename = trim_file_name(final_filename)
|
||||
# Get entry info json.
|
||||
options['outtmpl'] = str(final_filename)
|
||||
ydl = YoutubeDL(options)
|
||||
ydl.params['logger'] = ydl_logger
|
||||
ydl.add_default_info_extractors()
|
||||
entry = extract_info(url, ydl=ydl, process=True)
|
||||
else:
|
||||
raise
|
||||
|
||||
logger.debug(f'Downloading {url} to {repr(str(final_filename))}')
|
||||
return final_filename, entry
|
||||
|
|
|
|||
|
|
@ -12,34 +12,6 @@ if [ $EUID != 0 ]; then
|
|||
exit $?
|
||||
fi
|
||||
|
||||
Help() {
|
||||
# Display Help
|
||||
echo "Build WROLPi Raspberry Pi images"
|
||||
echo
|
||||
echo "Syntax: upgrade.sh [-h] [-b BRANCH]"
|
||||
echo "options:"
|
||||
echo "h Print this help."
|
||||
echo "b Build from this git BRANCH."
|
||||
echo
|
||||
}
|
||||
|
||||
BRANCH="master"
|
||||
while getopts ":hb:" option; do
|
||||
case $option in
|
||||
h) # display Help
|
||||
Help
|
||||
exit
|
||||
;;
|
||||
b)
|
||||
BRANCH="${OPTARG}"
|
||||
;;
|
||||
*) # invalid argument(s)
|
||||
echo "Error: Invalid option"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/gis-map.dump.gz" ]; then
|
||||
echo "stage2/04-wrolpi/files/gis-map.dump.gz does not exist!"
|
||||
exit 1
|
||||
|
|
@ -57,7 +29,6 @@ git clone --branch arm64 https://github.com/RPI-Distro/pi-gen.git "${BUILD_DIR}"
|
|||
|
||||
# Copy the configuration and build files into the pi-gen directory.
|
||||
cp "${SCRIPT_DIR}/config.txt" "${BUILD_DIR}/config.txt"
|
||||
echo "WROLPI_BRANCH=${BRANCH}" >> "${BUILD_DIR}/config.txt"
|
||||
rsync -a "${SCRIPT_DIR}"/stage2/* "${BUILD_DIR}/stage2/"
|
||||
|
||||
# We only need to build the Lite and Desktop images.
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ sed -i 's/port = 5433/port = 5432/' /etc/postgresql/15/main/postgresql.conf
|
|||
npm i -g serve@12.0.1 single-file-cli@1.1.54 readability-extractor@0.0.6 carto@1.2.0
|
||||
|
||||
# Put the latest WROLPi master in /opt/wrolpi.
|
||||
export WROLPI_BRANCH=${WROLPI_BRANCH:master}
|
||||
git clone -b ${WROLPI_BRANCH} https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi
|
||||
git clone -b master https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi
|
||||
git config --global --add safe.directory /opt/wrolpi
|
||||
|
||||
# Install Python requirements.
|
||||
|
|
|
|||
|
|
@ -476,7 +476,7 @@ class ConfigFile:
|
|||
logger_.debug(f'Copied backup config: {rel_path} -> {backup_file}')
|
||||
|
||||
# Write the config in-memory to the file. Track version changes to avoid overriding newer config.
|
||||
self._config['version'] += 1
|
||||
self._config['version'] = (self._config['version'] or 0) + 1
|
||||
config = deepcopy(self._config)
|
||||
self.write_config_data(config, file)
|
||||
|
||||
|
|
@ -1213,22 +1213,31 @@ def escape_file_name(name: str) -> str:
|
|||
|
||||
|
||||
# Maximum length is probably 255, but we need more length for large suffixes like `.readability.json`, and temporary
|
||||
# downloading suffixes.
|
||||
# downloading suffixes from yt-dlp.
|
||||
MAXIMUM_FILE_LENGTH = 180
|
||||
|
||||
|
||||
def trim_file_name(name: str) -> str:
|
||||
def trim_file_name(path: str | pathlib.Path) -> str | pathlib.Path:
|
||||
"""Shorten the file name only if it is longer than the file system supports. Trim from the end of the name until
|
||||
the name is short enough, excluding suffix."""
|
||||
if len(name) < MAXIMUM_FILE_LENGTH:
|
||||
return name
|
||||
the name is short enough (preserving any suffix)."""
|
||||
name_type = pathlib.Path if isinstance(path, pathlib.Path) else str
|
||||
parent = path.parent if isinstance(path, pathlib.Path) else '/'.join(i for i in path.split('/')[:-1])
|
||||
path = path.name if isinstance(path, pathlib.Path) else path
|
||||
|
||||
if len(path) < MAXIMUM_FILE_LENGTH:
|
||||
return name_type(path)
|
||||
|
||||
# Don't trim the filename to exactly 256 characters.
|
||||
# This is because a FileGroup will have varying filename lengths.
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
stem, suffix = split_path_stem_and_suffix(name)
|
||||
stem, suffix = split_path_stem_and_suffix(path)
|
||||
excess = MAXIMUM_FILE_LENGTH - len(suffix)
|
||||
return stem[:excess].strip() + suffix
|
||||
new_name = stem[:excess].strip() + suffix
|
||||
if parent:
|
||||
if name_type == pathlib.Path:
|
||||
return parent / new_name
|
||||
return f'{parent}/{new_name}'
|
||||
return name_type(new_name)
|
||||
|
||||
|
||||
def native_only(func: callable):
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ def strpdate(dt: str) -> datetime:
|
|||
if len(c) == 13:
|
||||
# d/m/Y HH:MM:SS
|
||||
return datetime.strptime(dt, '%m/%d/%Y %H:%M:%S')
|
||||
elif len(c) == 15:
|
||||
elif len(c) in (15, 16):
|
||||
# d/m/Y HH:MM:SS PM
|
||||
return datetime.strptime(dt, '%m/%d/%Y %H:%M:%S %p')
|
||||
elif dt.count('-') == 2 and len(dt) <= 10:
|
||||
|
|
|
|||
|
|
@ -1014,3 +1014,29 @@ async def test_config_valid(async_client, test_wrolpi_config):
|
|||
|
||||
test_wrolpi_config.write_text('wrol_mode: true')
|
||||
assert config.is_valid() is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize('name,expected_name', [
|
||||
('foo', 'foo'),
|
||||
(pathlib.Path('foo'), pathlib.Path('foo')),
|
||||
(
|
||||
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.txt',
|
||||
'this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.txt'
|
||||
),
|
||||
(
|
||||
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json',
|
||||
'some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
|
||||
),
|
||||
(
|
||||
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.info.json',
|
||||
'/absolute/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.info.json'
|
||||
),
|
||||
(
|
||||
pathlib.Path(
|
||||
'/some/parent/is/not/trimmed/this name is toooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo long.readability.json'),
|
||||
pathlib.Path(
|
||||
'/some/parent/is/not/trimmed/this name is tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.readability.json')
|
||||
),
|
||||
])
|
||||
def test_trim_file_name(name, expected_name):
|
||||
assert common.trim_file_name(name) == expected_name
|
||||
|
|
|
|||
|
|
@ -119,6 +119,7 @@ def test_timedelta_to_timestamp(td, expected):
|
|||
('2022-02-18T11:00:03.028Z', datetime(2022, 2, 18, 11, 0, 3, 28000, tzinfo=pytz.UTC)),
|
||||
# From SingleFile.
|
||||
('Fri Jun 17 2022 19:24:52', datetime(2022, 6, 17, 19, 24, 52)),
|
||||
('9/26/2024 11:55:13 AM', datetime(2024, 9, 26, 11, 55, 13)),
|
||||
# PDFs are the Wild West...
|
||||
("D:20221226113758-07'00", datetime(2022, 12, 26, 11, 37, 58, tzinfo=timezone(timedelta(days=-1, seconds=61200)))),
|
||||
('D:20200205184724', datetime(2020, 2, 5, 18, 47, 24)),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue