Bumping python requirements versions.

modules/videos/channel/api.py

@@ -166,4 +166,4 @@ async def channel_search(_: Request, body: schema.ChannelSearchRequest):
     ret = dict(
         channels=channels,
     )
-    return json_response(ret)
+    return json_response(ret)

modules/videos/channel/lib.py

@@ -171,7 +171,10 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
     except APIError as e:
         raise ValidationError() from e
 
-    pathlib.Path(data.directory).mkdir(parents=True, exist_ok=True)
+    directory = pathlib.Path(data.directory)
+    directory = get_media_directory() / directory if not directory.is_absolute() else directory
+    directory.mkdir(parents=True, exist_ok=True)
+
     channel = Channel()
     session.add(channel)
     session.flush([channel])

modules/videos/channel/test/test_api.py

@@ -316,7 +316,7 @@ async def test_channel_crud(test_session, async_client, test_directory, test_dow
 
     # Can't update channel that doesn't exist
     request, response = await async_client.put('/api/videos/channels/doesnt_exist',
-                                                    content=json.dumps(new_channel))
+                                               content=json.dumps(new_channel))
     assert response.status_code == HTTPStatus.NOT_FOUND
 
     # Delete the new channel

modules/videos/conftest.py

@@ -9,7 +9,6 @@ from uuid import uuid4
 import mock
 import pytest
 from PIL import Image
-from requests import session
 
 from modules.videos.downloader import VideoDownloader, ChannelDownloader
 from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination

modules/videos/lib.py

@@ -3,6 +3,7 @@ import dataclasses
 import html
 import pathlib
 import re
+from datetime import datetime
 from typing import Generator, Type
 from typing import Tuple, Optional
 
@@ -38,13 +39,14 @@ class VideoInfoJSON:
     channel_source_id: str | None = None
     channel_url: str | None = None
     duration: int | None = None
+    epoch: int | None = None
     title: str | None = None
+    upload_date: datetime | None = None
     url: str | None = None
     view_count: int | None = None
-    epoch: int | None = None
 
 
-def process_video_info_json(video: Video) -> VideoInfoJSON:
+def extract_video_info_json(video: Video) -> VideoInfoJSON:
     """
     Parse the Video's info json file, return the relevant data.
     """
@@ -53,12 +55,17 @@ def process_video_info_json(video: Video) -> VideoInfoJSON:
         title = info_json.get('fulltitle') or info_json.get('title')
         video_info_json.title = html.unescape(title) if title else None
 
-        video_info_json.duration = int(i) if (i := info_json.get('duration')) else None
-        video_info_json.view_count = int(i) if (i := info_json.get('view_count')) else None
-        video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None
+        upload_date = dates.strpdate(i) if (i := info_json.get('upload_date')) else None
+        if upload_date:
+            upload_date = upload_date.astimezone(pytz.UTC)
+
         video_info_json.channel_source_id = info_json.get('channel_id') or info_json.get('uploader_id') or None
         video_info_json.channel_url = info_json.get('channel_url') or info_json.get('uploader_url') or None
+        video_info_json.duration = int(i) if (i := info_json.get('duration')) else None
         video_info_json.epoch = int(i) if (i := info_json.get('epoch')) else None
+        video_info_json.upload_date = upload_date
+        video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None
+        video_info_json.view_count = int(i) if (i := info_json.get('view_count')) else None
 
     return video_info_json
 
@@ -76,15 +83,16 @@ def validate_video(video: Video, channel_generate_poster: bool):
     A Video is also valid when it has a JPEG poster, if any.  If no poster can be
     found, it will be generated from the video file.
     """
-    info_json_path = video.info_json_path
-    json_data_missing = bool(video.file_group.title) and bool(video.file_group.length) and bool(video.view_count) \
-                        and bool(video.file_group.url) and bool(video.file_group.download_datetime)
-    if info_json_path and json_data_missing is False:
+    json_published_datetime = None
+
+    if video.info_json_path:
         # These properties can be found in the info json.
-        video_info_json = process_video_info_json(video)
+        video_info_json = extract_video_info_json(video)
         video.file_group.title = video_info_json.title
         video.file_group.length = video_info_json.duration
         video.file_group.url = video_info_json.url
+        json_published_datetime = video.file_group.published_datetime = \
+            video_info_json.upload_date or video.file_group.published_datetime
         video.file_group.download_datetime = from_timestamp(video_info_json.epoch) if video_info_json.epoch else None
         # View count will probably be overwritten by more recent data when this Video's Channel is
         # updated.
@@ -117,7 +125,8 @@ def validate_video(video: Video, channel_generate_poster: bool):
             if not published_date.tzinfo:
                 published_date = published_date.astimezone(pytz.UTC)
         video.file_group.title = video.file_group.title or html.unescape(title)
-        video.file_group.published_datetime = published_date
+        # Trust info json upload_date before file name datetime.
+        video.file_group.published_datetime = json_published_datetime or published_date
         video.source_id = video.source_id or source_id
 
     if channel_generate_poster:

modules/videos/test/test_api.py

@@ -179,7 +179,7 @@ async def test_api_download_video_tags(test_session, async_client, tag_factory):
 
 
 @pytest.mark.asyncio
-async def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory,
+async def test_search_videos_file(test_session, test_directory, video_with_search_factory,
                                   assert_video_search):
     """Test that videos can be searched and that their order is by their textsearch rank."""
     # These captions have repeated letters, so they will be higher in the ranking.

modules/videos/test/test_lib.py

@@ -89,6 +89,25 @@ def test_validate_video(test_directory, video_factory, image_bytes_factory, vide
     assert vid1.source_id == '1234567890'
     assert vid1.file_group.title == 'The Title'
 
+    # Replace info json, FileGroup attributes should be replaced.
+    info_json = {
+        'channel_id': 'channel id',
+        'channel_url': 'https://example.com/example_video_json_channel',
+        'duration': 635,
+        'epoch': 123456789,
+        'fulltitle': 'The full title',
+        'id': 'some long id',
+        'title': 'The title',
+        'upload_date': '20240917',
+        'uploader_id': 'uploader id',
+        'uploader_url': 'https://example.com/example_video_json_uploader',
+        'view_count': 406
+    }
+    vid1.replace_info_json(info_json)
+    validate_video(vid1, False)
+    assert vid1.file_group.published_datetime and vid1.file_group.published_datetime.year == 2024
+    assert vid1.file_group.title == 'The full title'
+
     # A PNG is replaced.
     vid2 = video_factory(with_video_file=True, with_poster_ext='.png')
     vid2.poster_path.with_suffix('.jpg').write_bytes(image_bytes_factory())  # New poster exists, replace it.

modules/videos/video/api.py

@@ -6,7 +6,7 @@ from sanic_ext import validate
 from sanic_ext.extensions.openapi import openapi
 
 from wrolpi.api_utils import json_response
-from wrolpi.common import logger, wrol_mode_check, run_after
+from wrolpi.common import logger, wrol_mode_check
 from wrolpi.errors import InvalidOrderBy, ValidationError
 from wrolpi.events import Events
 from wrolpi.schema import JSONErrorResponse

modules/zim/models.py

@@ -12,7 +12,7 @@ from sqlalchemy.orm.exc import NoResultFound  # noqa
 
 from modules.zim.errors import UnknownZimEntry, UnknownZimTagEntry
 from wrolpi import dates, tags
-from wrolpi.common import Base, extract_html_text, logger, get_relative_to_media_directory
+from wrolpi.common import Base, logger, get_relative_to_media_directory
 from wrolpi.dates import TZDateTime
 from wrolpi.db import optional_session, get_db_curs
 from wrolpi.downloader import Download, download_manager

modules/zim/test/test_lib.py

@@ -250,7 +250,8 @@ async def test_zim_download(test_session, kiwix_download_zim, test_directory, te
 
 
 @pytest.mark.asyncio
-async def test_zim_tag_migration(async_client, test_session, test_directory, zim_factory, tag_factory, test_tags_config):
+async def test_zim_tag_migration(async_client, test_session, test_directory, zim_factory, tag_factory,
+                                 test_tags_config):
     """When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
     zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim')  # The outdated Zim.
     zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')

pytest.ini

@@ -3,3 +3,4 @@ asyncio_mode = auto
 testpaths =
     wrolpi
     modules
+asyncio_default_fixture_loop_scope = function

requirements.txt

@@ -1,25 +1,25 @@
-Pillow==10.3.0
-Pint==0.21
-PyYAML==6.0.1
+Pillow==10.4.0
+Pint==0.24.3
+PyYAML==6.0.2
 SQLAlchemy[asyncio]==1.3.22
-aiohttp~=3.9.5
+aiohttp==3.10.5
 alembic==1.10.4
-beautifulsoup4==4.12.2
-cachetools==5.3.0
+beautifulsoup4==4.12.3
+cachetools==5.5.0
 ebooklib==0.18
-feedparser==6.0.10
+feedparser==6.0.11
 jc==1.25.3
-libzim==3.3.0.post0
+libzim==3.5.0
 mock==5.1.0
-psutil==5.9.5
-psycopg2-binary==2.9.6
-pypdf==3.17.0
-pytest-asyncio==0.21.0
-pytest==7.3.1
-python-docx==1.1.0
-python-dotenv==1.0.0
+psutil==6.0.0
+psycopg2-binary==2.9.9
+pypdf==4.3.1
+pytest-asyncio==0.24.0
+pytest==8.3.3
+python-docx==1.1.2
+python-dotenv==1.0.1
 python-magic~=0.4.27
-pytz==2023.3
+pytz==2024.2
 sanic-ext==23.6.0
 sanic-testing==22.6.0  # 23.6.0 has error with "expandtabs"
 sanic==23.6.0  # This is the latest version of Sanic that avoids this bug: https://github.com/sanic-org/sanic/issues/2921
@@ -27,5 +27,5 @@ selenium==4.9.1
 srt==3.5.3
 vininfo[cli]==1.8.0
 websockets==12.0
-webvtt-py==0.4.6
+webvtt-py==0.5.1
 yt-dlp==2024.8.6

test/bad_caption.en.vtt

@@ -1,8 +0,0 @@
-WEBVTT
-Kind: captions
-Language: en
-
-00:00:00.070 --> 00:00:06.380
-this caption is bad
-
-00:05:08.650 --> 00:05:11.26

wrolpi/__init__.py

@@ -1,6 +1,6 @@
 import wrolpi.files.api  # noqa  Import from files so the blueprint is attached.
 import wrolpi.scrape_downloader  # noqa
-import wrolpi.status # noqa
+import wrolpi.status  # noqa
 
 BEFORE_STARTUP_FUNCTIONS = []
 

wrolpi/cmd.py

@@ -85,4 +85,4 @@ YT_DLP_BIN = which(
 )
 FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe')
 FFMPEG_BIN = which('ffmpeg', '/usr/bin/ffmpeg')
-JC_BIN = which('jc', '/usr/bin/jc')
+JC_BIN = which('jc', '/usr/bin/jc')

wrolpi/conftest.py

@@ -382,14 +382,6 @@ def srt_file3(test_directory) -> pathlib.Path:
     yield destination
 
 
-@pytest.fixture
-def bad_vtt_file(test_directory) -> pathlib.Path:
-    """Return a copy of the bad_caption VTT file in the `test_directory`."""
-    destination = test_directory / f'{uuid4()}.en.vtt'
-    shutil.copy(PROJECT_DIR / 'test/bad_caption.en.vtt', destination)
-    yield destination
-
-
 @pytest.fixture
 def example_pdf(test_directory) -> pathlib.Path:
     destination = test_directory / 'pdf example.pdf'

wrolpi/events.py

@@ -1,5 +1,3 @@
-import multiprocessing
-
 __all__ = ['Events', 'get_events']
 
 from datetime import datetime

wrolpi/tags.py

@@ -1,7 +1,6 @@
 import contextlib
 import pathlib
 from datetime import datetime
-from pprint import pprint
 from typing import List, Dict, Tuple, Optional
 
 from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger

wrolpi/test/test_captions.py

@@ -1,15 +1,8 @@
 from wrolpi import captions
 
 
-def test_bad_caption(bad_vtt_file):
-    """A caption file that cannot be parsed is ignored."""
-    result = captions.read_captions(bad_vtt_file)
-    assert result is None
-
-
 def test_get_caption_text(vtt_file1, srt_file3):
     expected = ['okay welcome to this session this is',
-                'okay welcome to this session this is',
                 'okay welcome to this session this is\ncalled the kinetic bunny need to meet',
                 'called the kinetic bunny need to meet',
                 'called the kinetic bunny need to meet\nthe virtual bonnie emilio stop this is',

wrolpi/test/test_downloader.py

@@ -5,7 +5,6 @@ from datetime import datetime
 from http import HTTPStatus
 from itertools import zip_longest
 from unittest import mock
-from unittest.mock import AsyncMock
 
 import pytest
 import pytz

wrolpi/test/test_root_api.py

@@ -5,7 +5,6 @@ from itertools import zip_longest
 import pytest
 from mock import mock
 from sanic import Request
-from selenium.webdriver.common.devtools.v112.runtime import await_promise
 
 from wrolpi.admin import HotspotStatus
 from wrolpi.api_utils import json_error_handler

wrolpi/vars.py

@@ -49,6 +49,7 @@ LOG_LEVEL_INT = logging.INFO
 if '-vv' in sys.argv:
     LOG_LEVEL_INT = logging.DEBUG
 elif '-vvv' in sys.argv:
+    # SQLAlchemy will increase logging.
     LOG_LEVEL_INT = logging.NOTSET
 LOG_LEVEL = logging.getLevelName(LOG_LEVEL_INT)