Including Country/Region codes in caption file names.

This commit is contained in:
Roland 2024-08-21 21:19:39 -06:00
parent 0929bac69e
commit d5ca3e7f02
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
12 changed files with 280 additions and 23 deletions

View file

@ -207,7 +207,7 @@ export function StatusPage() {
if (status && status['cpu_stats']) {
const {cpu_stats, load_stats, memory_stats, processes_stats} = status;
percent = cpu_stats['percent'];
cores = cpu_stats['cores'];
cores = cpu_stats['cores'] || '?';
temperature = cpu_stats['temperature'];
high_temperature = cpu_stats['high_temperature'];
critical_temperature = cpu_stats['critical_temperature'];

View file

@ -71,7 +71,8 @@ class ArchiveDownloader(Downloader, ABC):
async def do_singlefile(self, download: Download) -> Tuple[bytes, dict]:
"""Create a Singlefile from the archive's URL."""
cmd = (str(SINGLE_FILE_BIN),
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
str(SINGLE_FILE_BIN),
download.url,
'--browser-executable-path', CHROMIUM,
'--browser-args', '["--no-sandbox"]',

View file

@ -53,8 +53,6 @@ def channel_get(_: Request, channel_id: int = None):
@wrol_mode_check
def channel_post(_: Request, body: schema.ChannelPostRequest):
body.directory = get_media_directory() / body.directory
if not body.directory.is_dir() and body.mkdir:
make_media_directory(body.directory)
channel = lib.create_channel(data=body, return_dict=False)

View file

@ -171,6 +171,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
except APIError as e:
raise ValidationError() from e
pathlib.Path(data.directory).mkdir(parents=True, exist_ok=True)
channel = Channel()
session.add(channel)
session.flush([channel])

View file

@ -249,15 +249,6 @@ def test_channel_post_directory(test_session, test_client, test_directory):
assert response.status_code == HTTPStatus.CREATED
directory = test_session.query(Channel).filter_by(id=1).one().directory
assert (test_directory / 'foo') == directory
assert not directory.is_dir()
assert directory.is_absolute()
# Channel can be created and have its directory be created.
data = dict(name='bar', directory='bar', mkdir=True)
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
assert response.status_code == HTTPStatus.CREATED
directory = test_session.query(Channel).filter_by(id=2).one().directory
assert (test_directory / 'bar') == directory
assert directory.is_dir()
assert directory.is_absolute()

View file

@ -101,9 +101,9 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact
Channels can be searched by their name.
"""
# Create four channels with varying associated videos.
c1 = channel_factory(name='foo')
c1 = channel_factory(name='Foo')
c2 = channel_factory(name='bar')
c3 = channel_factory(name='foo bar')
c3 = channel_factory(name='Foo Bar')
c4 = channel_factory(name='foobar')
video_factory(channel_id=c1.id)
for _ in range(2):
@ -123,8 +123,8 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact
if channel_names:
raise AssertionError('More channels than expected.')
await assert_search('foo', ['foo', 'foo bar', 'foobar'])
await assert_search('bar', ['foobar', 'foo bar', 'bar'], True)
await assert_search('foo', ['Foo', 'foobar', 'Foo Bar'])
await assert_search('bar', ['foobar', 'Foo Bar', 'bar'], True)
@pytest.mark.asyncio

View file

@ -33,7 +33,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
NoPrimaryFile, InvalidDirectory
from wrolpi.events import Events
from wrolpi.files.models import FileGroup, Directory
from wrolpi.lang import ISO_639_CODES
from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config
from wrolpi.vars import PYTEST
@ -277,6 +277,10 @@ SUFFIXES = {
}
SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES}
SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES}
EXTRA_SUFFIXES = set()
for six_ in ISO_639_CODES.keys():
EXTRA_SUFFIXES |= {f'.{six_}-{i}.srt' for i in ISO_3166_CODES}
EXTRA_SUFFIXES |= {f'.{six_}-{i}.vtt' for i in ISO_3166_CODES}
@functools.lru_cache(maxsize=10_000)
@ -297,7 +301,12 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = Fals
# May or may not be absolute. Convert to lowercase so any suffix case can be matched.
full_ = str(path).lower()
# Get the special matching suffix, if any. Match against `.en.srt` but could return `.EN.SRT`
suffix = next(filter(lambda i: full_.endswith(i), SUFFIXES), path.suffix)
suffix = next(filter(lambda i: full_.endswith(i), SUFFIXES), None)
if not suffix and (full_.endswith('.srt') or full_.endswith('.vtt')):
# Special handling for numerous language/region codes.
suffix = next(filter(lambda i: full_.endswith(i), EXTRA_SUFFIXES), path.suffix)
# Fallback to pathlib's suffix.
suffix = suffix or path.suffix
# Return the suffix from the path's name in the original case.
if suffix:
idx = -1 * len(suffix)

View file

@ -145,6 +145,9 @@ async def test_delete_nested(test_session, make_files_structure):
('/absolute/foo.bar', False, ('foo', '.bar')),
('foo.en.srt', False, ('foo', '.en.srt')),
('foo.pl.srt', False, ('foo', '.pl.srt')),
('foo.en-US.srt', False, ('foo', '.en-US.srt')),
('foo.en-US.vtt', False, ('foo', '.en-US.vtt')),
('foo.en-us.vtt', False, ('foo', '.en-us.vtt')),
# Absolute path can be returned.
('/absolute//foo.bar', True, ('/absolute/foo', '.bar')),
# Case is preserved.

View file

@ -672,3 +672,255 @@ ISO_639_CODES = {
"zxx": "No linguistic content; Not applicable",
"zza": "Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki",
}
ISO_3166_CODES = {
'af': 'Afghanistan[b]',
'ax': 'Åland Islands',
'al': 'Albania',
'dz': 'Algeria',
'as': 'American Samoa',
'ad': 'Andorra',
'ao': 'Angola',
'ai': 'Anguilla',
'aq': 'Antarctica',
'ag': 'Antigua and Barbuda',
'ar': 'Argentina',
'am': 'Armenia',
'aw': 'Aruba',
'au': 'Australia',
'at': 'Austria',
'az': 'Azerbaijan',
'bs': 'Bahamas',
'bh': 'Bahrain',
'bd': 'Bangladesh',
'bb': 'Barbados',
'by': 'Belarus',
'be': 'Belgium',
'bz': 'Belize',
'bj': 'Benin',
'bm': 'Bermuda',
'bt': 'Bhutan',
'bo': 'Bolivia, Plurinational State of',
'bq': 'Bonaire, Sint Eustatius and Saba[c]',
'ba': 'Bosnia and Herzegovina',
'bw': 'Botswana',
'bv': 'Bouvet Island',
'br': 'Brazil',
'io': 'British Indian Ocean Territory',
'bn': 'Brunei Darussalam',
'bg': 'Bulgaria',
'bf': 'Burkina Faso',
'bi': 'Burundi',
'cv': 'Cabo Verde',
'kh': 'Cambodia',
'cm': 'Cameroon',
'ca': 'Canada',
'ky': 'Cayman Islands',
'cf': 'Central African Republic',
'td': 'Chad',
'cl': 'Chile',
'cn': 'China[b]',
'cx': 'Christmas Island',
'cc': 'Cocos (Keeling) Islands',
'co': 'Colombia',
'km': 'Comoros',
'cg': 'Congo',
'cd': 'Congo, Democratic Republic of the',
'ck': 'Cook Islands',
'cr': 'Costa Rica',
'ci': "Côte d'Ivoire",
'hr': 'Croatia',
'cu': 'Cuba',
'cw': 'Curaçao',
'cy': 'Cyprus[b]',
'cz': 'Czechia',
'dk': 'Denmark',
'dj': 'Djibouti',
'dm': 'Dominica',
'do': 'Dominican Republic',
'ec': 'Ecuador',
'eg': 'Egypt',
'sv': 'El Salvador',
'gq': 'Equatorial Guinea',
'er': 'Eritrea',
'ee': 'Estonia',
'sz': 'Eswatini',
'et': 'Ethiopia',
'fk': 'Falkland Islands (Malvinas)[b]',
'fo': 'Faroe Islands',
'fj': 'Fiji',
'fi': 'Finland',
'fr': 'France',
'gf': 'French Guiana',
'pf': 'French Polynesia',
'tf': 'French Southern Territories',
'ga': 'Gabon',
'gm': 'Gambia',
'ge': 'Georgia',
'de': 'Germany',
'gh': 'Ghana',
'gi': 'Gibraltar',
'gr': 'Greece',
'gl': 'Greenland',
'gd': 'Grenada',
'gp': 'Guadeloupe',
'gu': 'Guam',
'gt': 'Guatemala',
'gg': 'Guernsey',
'gn': 'Guinea',
'gw': 'Guinea-Bissau',
'gy': 'Guyana',
'ht': 'Haiti',
'hm': 'Heard Island and McDonald Islands',
'va': 'Holy See',
'hn': 'Honduras',
'hk': 'Hong Kong',
'hu': 'Hungary',
'is': 'Iceland',
'in': 'India',
'id': 'Indonesia',
'ir': 'Iran, Islamic Republic of',
'iq': 'Iraq',
'ie': 'Ireland',
'im': 'Isle of Man',
'il': 'Israel',
'it': 'Italy',
'jm': 'Jamaica',
'jp': 'Japan',
'je': 'Jersey',
'jo': 'Jordan',
'kz': 'Kazakhstan',
'ke': 'Kenya',
'ki': 'Kiribati',
'kp': "Korea, Democratic People's Republic of",
'kr': 'Korea, Republic of',
'kw': 'Kuwait',
'kg': 'Kyrgyzstan',
'la': "Lao People's Democratic Republic",
'lv': 'Latvia',
'lb': 'Lebanon',
'ls': 'Lesotho',
'lr': 'Liberia',
'ly': 'Libya',
'li': 'Liechtenstein',
'lt': 'Lithuania',
'lu': 'Luxembourg',
'mo': 'Macao',
'mg': 'Madagascar',
'mw': 'Malawi',
'my': 'Malaysia',
'mv': 'Maldives',
'ml': 'Mali',
'mt': 'Malta',
'mh': 'Marshall Islands',
'mq': 'Martinique',
'mr': 'Mauritania',
'mu': 'Mauritius',
'yt': 'Mayotte',
'mx': 'Mexico',
'fm': 'Micronesia, Federated States of',
'md': 'Moldova, Republic of',
'mc': 'Monaco',
'mn': 'Mongolia',
'me': 'Montenegro',
'ms': 'Montserrat',
'ma': 'Morocco',
'mz': 'Mozambique',
'mm': 'Myanmar',
'na': 'Namibia',
'nr': 'Nauru',
'np': 'Nepal',
'nl': 'Netherlands, Kingdom of the',
'nc': 'New Caledonia',
'nz': 'New Zealand',
'ni': 'Nicaragua',
'ne': 'Niger',
'ng': 'Nigeria',
'nu': 'Niue',
'nf': 'Norfolk Island',
'mk': 'North Macedonia',
'mp': 'Northern Mariana Islands',
'no': 'Norway',
'om': 'Oman',
'pk': 'Pakistan',
'pw': 'Palau',
'ps': 'Palestine, State of[b]',
'pa': 'Panama',
'pg': 'Papua New Guinea',
'py': 'Paraguay',
'pe': 'Peru',
'ph': 'Philippines',
'pn': 'Pitcairn',
'pl': 'Poland',
'pt': 'Portugal',
'pr': 'Puerto Rico',
'qa': 'Qatar',
're': 'Réunion',
'ro': 'Romania',
'ru': 'Russian Federation',
'rw': 'Rwanda',
'bl': 'Saint Barthélemy',
'sh': 'Saint Helena, Ascension and Tristan da Cunha[d]',
'kn': 'Saint Kitts and Nevis',
'lc': 'Saint Lucia',
'mf': 'Saint Martin (French part)',
'pm': 'Saint Pierre and Miquelon',
'vc': 'Saint Vincent and the Grenadines',
'ws': 'Samoa',
'sm': 'San Marino',
'st': 'Sao Tome and Principe',
'sa': 'Saudi Arabia',
'sn': 'Senegal',
'rs': 'Serbia',
'sc': 'Seychelles',
'sl': 'Sierra Leone',
'sg': 'Singapore',
'sx': 'Sint Maarten (Dutch part)',
'sk': 'Slovakia',
'si': 'Slovenia',
'sb': 'Solomon Islands',
'so': 'Somalia',
'za': 'South Africa',
'gs': 'South Georgia and the South Sandwich Islands',
'ss': 'South Sudan',
'es': 'Spain',
'lk': 'Sri Lanka',
'sd': 'Sudan',
'sr': 'Suriname',
'sj': 'Svalbard and Jan Mayen[e]',
'se': 'Sweden',
'ch': 'Switzerland',
'sy': 'Syrian Arab Republic',
'tw': 'Taiwan, Province of China[b]',
'tj': 'Tajikistan',
'tz': 'Tanzania, United Republic of',
'th': 'Thailand',
'tl': 'Timor-Leste',
'tg': 'Togo',
'tk': 'Tokelau',
'to': 'Tonga',
'tt': 'Trinidad and Tobago',
'tn': 'Tunisia',
'tr': 'Türkiye',
'tm': 'Turkmenistan',
'tc': 'Turks and Caicos Islands',
'tv': 'Tuvalu',
'ug': 'Uganda',
'ua': 'Ukraine',
'ae': 'United Arab Emirates',
'gb': 'United Kingdom of Great Britain and Northern Ireland',
'us': 'United States of America',
'um': 'United States Minor Outlying Islands[f]',
'uy': 'Uruguay',
'uz': 'Uzbekistan',
'vu': 'Vanuatu',
've': 'Venezuela, Bolivarian Republic of',
'vn': 'Viet Nam',
'vg': 'Virgin Islands (British)',
'vi': 'Virgin Islands (U.S.)',
'wf': 'Wallis and Futuna',
'eh': 'Western Sahara[b]',
'ye': 'Yemen',
'zm': 'Zambia',
'zw': 'Zimbabwe',
}

View file

@ -100,6 +100,8 @@ class DownloadSettings:
raise ValidationError('suffix must start with .')
self.title_exclude = self.title_exclude or None
self.title_include = self.title_include or None
if not self.download_metadata_only:
del self.download_metadata_only
@dataclass
@ -109,7 +111,6 @@ class DownloadRequest:
frequency: Optional[int] = None
sub_downloader: Optional[str] = None
settings: Optional[dict] = field(default_factory=lambda: dict())
download_metadata_only: Optional[bool] = False
def __post_init__(self):
urls = [j for i in self.urls if (j := i.strip())]

View file

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import asyncio
import functools
import json
import multiprocessing
import pathlib
@ -466,8 +467,8 @@ async def get_disk_counters(shared_status):
bps = _calculate_bytes_per_second(old_stats, new_stats)
new_stats.update(dict(
elapsed=bps['elapsed'],
bytes_read_ps = bps['bytes_recv_ps'],
bytes_write_ps = bps['bytes_sent_ps'],
bytes_read_ps=bps['bytes_recv_ps'],
bytes_write_ps=bps['bytes_sent_ps'],
max_read_ps=max(bps['bytes_recv_ps'], old_stats['max_read_ps']),
max_write_ps=max(bps['bytes_sent_ps'], old_stats['max_write_ps']),
))

View file

@ -54,4 +54,4 @@ async def test_status_worker(test_async_client):
assert api_app.shared_ctx.status['nic_bandwidth_stats']
assert 'bytes_recv' in list(api_app.shared_ctx.status['nic_bandwidth_stats'].values())[0]
assert api_app.shared_ctx.status['disk_bandwidth_stats']
assert 'bytes_recv' in list(api_app.shared_ctx.status['disk_bandwidth_stats'].values())[0]
assert 'bytes_read' in list(api_app.shared_ctx.status['disk_bandwidth_stats'].values())[0]