From d5ca3e7f02f798f2526b541c9ebb45d0bb56b9d6 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 21 Aug 2024 21:19:39 -0600 Subject: [PATCH] Including Country/Region codes in caption file names. --- app/src/components/admin/Status.js | 2 +- modules/archive/__init__.py | 3 +- modules/videos/channel/api.py | 2 - modules/videos/channel/lib.py | 1 + modules/videos/channel/test/test_api.py | 9 - modules/videos/channel/test/test_lib.py | 8 +- wrolpi/files/lib.py | 13 +- wrolpi/files/test/test_lib.py | 3 + wrolpi/lang.py | 252 ++++++++++++++++++++++++ wrolpi/schema.py | 3 +- wrolpi/status.py | 5 +- wrolpi/test/test_status.py | 2 +- 12 files changed, 280 insertions(+), 23 deletions(-) diff --git a/app/src/components/admin/Status.js b/app/src/components/admin/Status.js index 328b2446..994e3b48 100644 --- a/app/src/components/admin/Status.js +++ b/app/src/components/admin/Status.js @@ -207,7 +207,7 @@ export function StatusPage() { if (status && status['cpu_stats']) { const {cpu_stats, load_stats, memory_stats, processes_stats} = status; percent = cpu_stats['percent']; - cores = cpu_stats['cores']; + cores = cpu_stats['cores'] || '?'; temperature = cpu_stats['temperature']; high_temperature = cpu_stats['high_temperature']; critical_temperature = cpu_stats['critical_temperature']; diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 29eeaa4b..fb0ed6db 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -71,7 +71,8 @@ class ArchiveDownloader(Downloader, ABC): async def do_singlefile(self, download: Download) -> Tuple[bytes, dict]: """Create a Singlefile from the archive's URL.""" - cmd = (str(SINGLE_FILE_BIN), + cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority. + str(SINGLE_FILE_BIN), download.url, '--browser-executable-path', CHROMIUM, '--browser-args', '["--no-sandbox"]', diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index 72b6db22..0cb962f3 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -53,8 +53,6 @@ def channel_get(_: Request, channel_id: int = None): @wrol_mode_check def channel_post(_: Request, body: schema.ChannelPostRequest): body.directory = get_media_directory() / body.directory - if not body.directory.is_dir() and body.mkdir: - make_media_directory(body.directory) channel = lib.create_channel(data=body, return_dict=False) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 4aa38cb4..a28e9b20 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -171,6 +171,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic except APIError as e: raise ValidationError() from e + pathlib.Path(data.directory).mkdir(parents=True, exist_ok=True) channel = Channel() session.add(channel) session.flush([channel]) diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index 440eaf9c..c71aa872 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -249,15 +249,6 @@ def test_channel_post_directory(test_session, test_client, test_directory): assert response.status_code == HTTPStatus.CREATED directory = test_session.query(Channel).filter_by(id=1).one().directory assert (test_directory / 'foo') == directory - assert not directory.is_dir() - assert directory.is_absolute() - - # Channel can be created and have its directory be created. - data = dict(name='bar', directory='bar', mkdir=True) - request, response = test_client.post('/api/videos/channels', content=json.dumps(data)) - assert response.status_code == HTTPStatus.CREATED - directory = test_session.query(Channel).filter_by(id=2).one().directory - assert (test_directory / 'bar') == directory assert directory.is_dir() assert directory.is_absolute() diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py index 3a36d021..d68b7963 100644 --- a/modules/videos/channel/test/test_lib.py +++ b/modules/videos/channel/test/test_lib.py @@ -101,9 +101,9 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact Channels can be searched by their name. """ # Create four channels with varying associated videos. - c1 = channel_factory(name='foo') + c1 = channel_factory(name='Foo') c2 = channel_factory(name='bar') - c3 = channel_factory(name='foo bar') + c3 = channel_factory(name='Foo Bar') c4 = channel_factory(name='foobar') video_factory(channel_id=c1.id) for _ in range(2): @@ -123,8 +123,8 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact if channel_names: raise AssertionError('More channels than expected.') - await assert_search('foo', ['foo', 'foo bar', 'foobar']) - await assert_search('bar', ['foobar', 'foo bar', 'bar'], True) + await assert_search('foo', ['Foo', 'foobar', 'Foo Bar']) + await assert_search('bar', ['foobar', 'Foo Bar', 'bar'], True) @pytest.mark.asyncio diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 5f9af1d1..601737b5 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -33,7 +33,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag NoPrimaryFile, InvalidDirectory from wrolpi.events import Events from wrolpi.files.models import FileGroup, Directory -from wrolpi.lang import ISO_639_CODES +from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config from wrolpi.vars import PYTEST @@ -277,6 +277,10 @@ SUFFIXES = { } SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES} SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES} +EXTRA_SUFFIXES = set() +for six_ in ISO_639_CODES.keys(): + EXTRA_SUFFIXES |= {f'.{six_}-{i}.srt' for i in ISO_3166_CODES} + EXTRA_SUFFIXES |= {f'.{six_}-{i}.vtt' for i in ISO_3166_CODES} @functools.lru_cache(maxsize=10_000) @@ -297,7 +301,12 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = Fals # May or may not be absolute. Convert to lowercase so any suffix case can be matched. full_ = str(path).lower() # Get the special matching suffix, if any. Match against `.en.srt` but could return `.EN.SRT` - suffix = next(filter(lambda i: full_.endswith(i), SUFFIXES), path.suffix) + suffix = next(filter(lambda i: full_.endswith(i), SUFFIXES), None) + if not suffix and (full_.endswith('.srt') or full_.endswith('.vtt')): + # Special handling for numerous language/region codes. + suffix = next(filter(lambda i: full_.endswith(i), EXTRA_SUFFIXES), path.suffix) + # Fallback to pathlib's suffix. + suffix = suffix or path.suffix # Return the suffix from the path's name in the original case. if suffix: idx = -1 * len(suffix) diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 4ca75dc4..5f402c64 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -145,6 +145,9 @@ async def test_delete_nested(test_session, make_files_structure): ('/absolute/foo.bar', False, ('foo', '.bar')), ('foo.en.srt', False, ('foo', '.en.srt')), ('foo.pl.srt', False, ('foo', '.pl.srt')), + ('foo.en-US.srt', False, ('foo', '.en-US.srt')), + ('foo.en-US.vtt', False, ('foo', '.en-US.vtt')), + ('foo.en-us.vtt', False, ('foo', '.en-us.vtt')), # Absolute path can be returned. ('/absolute//foo.bar', True, ('/absolute/foo', '.bar')), # Case is preserved. diff --git a/wrolpi/lang.py b/wrolpi/lang.py index fd19c81b..52a92f98 100644 --- a/wrolpi/lang.py +++ b/wrolpi/lang.py @@ -672,3 +672,255 @@ ISO_639_CODES = { "zxx": "No linguistic content; Not applicable", "zza": "Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki", } + +ISO_3166_CODES = { + 'af': 'Afghanistan[b]', + 'ax': 'Åland Islands', + 'al': 'Albania', + 'dz': 'Algeria', + 'as': 'American Samoa', + 'ad': 'Andorra', + 'ao': 'Angola', + 'ai': 'Anguilla', + 'aq': 'Antarctica', + 'ag': 'Antigua and Barbuda', + 'ar': 'Argentina', + 'am': 'Armenia', + 'aw': 'Aruba', + 'au': 'Australia', + 'at': 'Austria', + 'az': 'Azerbaijan', + 'bs': 'Bahamas', + 'bh': 'Bahrain', + 'bd': 'Bangladesh', + 'bb': 'Barbados', + 'by': 'Belarus', + 'be': 'Belgium', + 'bz': 'Belize', + 'bj': 'Benin', + 'bm': 'Bermuda', + 'bt': 'Bhutan', + 'bo': 'Bolivia, Plurinational State of', + 'bq': 'Bonaire, Sint Eustatius and Saba[c]', + 'ba': 'Bosnia and Herzegovina', + 'bw': 'Botswana', + 'bv': 'Bouvet Island', + 'br': 'Brazil', + 'io': 'British Indian Ocean Territory', + 'bn': 'Brunei Darussalam', + 'bg': 'Bulgaria', + 'bf': 'Burkina Faso', + 'bi': 'Burundi', + 'cv': 'Cabo Verde', + 'kh': 'Cambodia', + 'cm': 'Cameroon', + 'ca': 'Canada', + 'ky': 'Cayman Islands', + 'cf': 'Central African Republic', + 'td': 'Chad', + 'cl': 'Chile', + 'cn': 'China[b]', + 'cx': 'Christmas Island', + 'cc': 'Cocos (Keeling) Islands', + 'co': 'Colombia', + 'km': 'Comoros', + 'cg': 'Congo', + 'cd': 'Congo, Democratic Republic of the', + 'ck': 'Cook Islands', + 'cr': 'Costa Rica', + 'ci': "Côte d'Ivoire", + 'hr': 'Croatia', + 'cu': 'Cuba', + 'cw': 'Curaçao', + 'cy': 'Cyprus[b]', + 'cz': 'Czechia', + 'dk': 'Denmark', + 'dj': 'Djibouti', + 'dm': 'Dominica', + 'do': 'Dominican Republic', + 'ec': 'Ecuador', + 'eg': 'Egypt', + 'sv': 'El Salvador', + 'gq': 'Equatorial Guinea', + 'er': 'Eritrea', + 'ee': 'Estonia', + 'sz': 'Eswatini', + 'et': 'Ethiopia', + 'fk': 'Falkland Islands (Malvinas)[b]', + 'fo': 'Faroe Islands', + 'fj': 'Fiji', + 'fi': 'Finland', + 'fr': 'France', + 'gf': 'French Guiana', + 'pf': 'French Polynesia', + 'tf': 'French Southern Territories', + 'ga': 'Gabon', + 'gm': 'Gambia', + 'ge': 'Georgia', + 'de': 'Germany', + 'gh': 'Ghana', + 'gi': 'Gibraltar', + 'gr': 'Greece', + 'gl': 'Greenland', + 'gd': 'Grenada', + 'gp': 'Guadeloupe', + 'gu': 'Guam', + 'gt': 'Guatemala', + 'gg': 'Guernsey', + 'gn': 'Guinea', + 'gw': 'Guinea-Bissau', + 'gy': 'Guyana', + 'ht': 'Haiti', + 'hm': 'Heard Island and McDonald Islands', + 'va': 'Holy See', + 'hn': 'Honduras', + 'hk': 'Hong Kong', + 'hu': 'Hungary', + 'is': 'Iceland', + 'in': 'India', + 'id': 'Indonesia', + 'ir': 'Iran, Islamic Republic of', + 'iq': 'Iraq', + 'ie': 'Ireland', + 'im': 'Isle of Man', + 'il': 'Israel', + 'it': 'Italy', + 'jm': 'Jamaica', + 'jp': 'Japan', + 'je': 'Jersey', + 'jo': 'Jordan', + 'kz': 'Kazakhstan', + 'ke': 'Kenya', + 'ki': 'Kiribati', + 'kp': "Korea, Democratic People's Republic of", + 'kr': 'Korea, Republic of', + 'kw': 'Kuwait', + 'kg': 'Kyrgyzstan', + 'la': "Lao People's Democratic Republic", + 'lv': 'Latvia', + 'lb': 'Lebanon', + 'ls': 'Lesotho', + 'lr': 'Liberia', + 'ly': 'Libya', + 'li': 'Liechtenstein', + 'lt': 'Lithuania', + 'lu': 'Luxembourg', + 'mo': 'Macao', + 'mg': 'Madagascar', + 'mw': 'Malawi', + 'my': 'Malaysia', + 'mv': 'Maldives', + 'ml': 'Mali', + 'mt': 'Malta', + 'mh': 'Marshall Islands', + 'mq': 'Martinique', + 'mr': 'Mauritania', + 'mu': 'Mauritius', + 'yt': 'Mayotte', + 'mx': 'Mexico', + 'fm': 'Micronesia, Federated States of', + 'md': 'Moldova, Republic of', + 'mc': 'Monaco', + 'mn': 'Mongolia', + 'me': 'Montenegro', + 'ms': 'Montserrat', + 'ma': 'Morocco', + 'mz': 'Mozambique', + 'mm': 'Myanmar', + 'na': 'Namibia', + 'nr': 'Nauru', + 'np': 'Nepal', + 'nl': 'Netherlands, Kingdom of the', + 'nc': 'New Caledonia', + 'nz': 'New Zealand', + 'ni': 'Nicaragua', + 'ne': 'Niger', + 'ng': 'Nigeria', + 'nu': 'Niue', + 'nf': 'Norfolk Island', + 'mk': 'North Macedonia', + 'mp': 'Northern Mariana Islands', + 'no': 'Norway', + 'om': 'Oman', + 'pk': 'Pakistan', + 'pw': 'Palau', + 'ps': 'Palestine, State of[b]', + 'pa': 'Panama', + 'pg': 'Papua New Guinea', + 'py': 'Paraguay', + 'pe': 'Peru', + 'ph': 'Philippines', + 'pn': 'Pitcairn', + 'pl': 'Poland', + 'pt': 'Portugal', + 'pr': 'Puerto Rico', + 'qa': 'Qatar', + 're': 'Réunion', + 'ro': 'Romania', + 'ru': 'Russian Federation', + 'rw': 'Rwanda', + 'bl': 'Saint Barthélemy', + 'sh': 'Saint Helena, Ascension and Tristan da Cunha[d]', + 'kn': 'Saint Kitts and Nevis', + 'lc': 'Saint Lucia', + 'mf': 'Saint Martin (French part)', + 'pm': 'Saint Pierre and Miquelon', + 'vc': 'Saint Vincent and the Grenadines', + 'ws': 'Samoa', + 'sm': 'San Marino', + 'st': 'Sao Tome and Principe', + 'sa': 'Saudi Arabia', + 'sn': 'Senegal', + 'rs': 'Serbia', + 'sc': 'Seychelles', + 'sl': 'Sierra Leone', + 'sg': 'Singapore', + 'sx': 'Sint Maarten (Dutch part)', + 'sk': 'Slovakia', + 'si': 'Slovenia', + 'sb': 'Solomon Islands', + 'so': 'Somalia', + 'za': 'South Africa', + 'gs': 'South Georgia and the South Sandwich Islands', + 'ss': 'South Sudan', + 'es': 'Spain', + 'lk': 'Sri Lanka', + 'sd': 'Sudan', + 'sr': 'Suriname', + 'sj': 'Svalbard and Jan Mayen[e]', + 'se': 'Sweden', + 'ch': 'Switzerland', + 'sy': 'Syrian Arab Republic', + 'tw': 'Taiwan, Province of China[b]', + 'tj': 'Tajikistan', + 'tz': 'Tanzania, United Republic of', + 'th': 'Thailand', + 'tl': 'Timor-Leste', + 'tg': 'Togo', + 'tk': 'Tokelau', + 'to': 'Tonga', + 'tt': 'Trinidad and Tobago', + 'tn': 'Tunisia', + 'tr': 'Türkiye', + 'tm': 'Turkmenistan', + 'tc': 'Turks and Caicos Islands', + 'tv': 'Tuvalu', + 'ug': 'Uganda', + 'ua': 'Ukraine', + 'ae': 'United Arab Emirates', + 'gb': 'United Kingdom of Great Britain and Northern Ireland', + 'us': 'United States of America', + 'um': 'United States Minor Outlying Islands[f]', + 'uy': 'Uruguay', + 'uz': 'Uzbekistan', + 'vu': 'Vanuatu', + 've': 'Venezuela, Bolivarian Republic of', + 'vn': 'Viet Nam', + 'vg': 'Virgin Islands (British)', + 'vi': 'Virgin Islands (U.S.)', + 'wf': 'Wallis and Futuna', + 'eh': 'Western Sahara[b]', + 'ye': 'Yemen', + 'zm': 'Zambia', + 'zw': 'Zimbabwe', +} diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 3f3d1104..bde4862d 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -100,6 +100,8 @@ class DownloadSettings: raise ValidationError('suffix must start with .') self.title_exclude = self.title_exclude or None self.title_include = self.title_include or None + if not self.download_metadata_only: + del self.download_metadata_only @dataclass @@ -109,7 +111,6 @@ class DownloadRequest: frequency: Optional[int] = None sub_downloader: Optional[str] = None settings: Optional[dict] = field(default_factory=lambda: dict()) - download_metadata_only: Optional[bool] = False def __post_init__(self): urls = [j for i in self.urls if (j := i.strip())] diff --git a/wrolpi/status.py b/wrolpi/status.py index b4da632e..6b3ecc8f 100755 --- a/wrolpi/status.py +++ b/wrolpi/status.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import asyncio +import functools import json import multiprocessing import pathlib @@ -466,8 +467,8 @@ async def get_disk_counters(shared_status): bps = _calculate_bytes_per_second(old_stats, new_stats) new_stats.update(dict( elapsed=bps['elapsed'], - bytes_read_ps = bps['bytes_recv_ps'], - bytes_write_ps = bps['bytes_sent_ps'], + bytes_read_ps=bps['bytes_recv_ps'], + bytes_write_ps=bps['bytes_sent_ps'], max_read_ps=max(bps['bytes_recv_ps'], old_stats['max_read_ps']), max_write_ps=max(bps['bytes_sent_ps'], old_stats['max_write_ps']), )) diff --git a/wrolpi/test/test_status.py b/wrolpi/test/test_status.py index 7cdb4c0a..12cfc84e 100644 --- a/wrolpi/test/test_status.py +++ b/wrolpi/test/test_status.py @@ -54,4 +54,4 @@ async def test_status_worker(test_async_client): assert api_app.shared_ctx.status['nic_bandwidth_stats'] assert 'bytes_recv' in list(api_app.shared_ctx.status['nic_bandwidth_stats'].values())[0] assert api_app.shared_ctx.status['disk_bandwidth_stats'] - assert 'bytes_recv' in list(api_app.shared_ctx.status['disk_bandwidth_stats'].values())[0] + assert 'bytes_read' in list(api_app.shared_ctx.status['disk_bandwidth_stats'].values())[0]