diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index 80b80aeb..58056446 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -166,4 +166,4 @@ async def channel_search(_: Request, body: schema.ChannelSearchRequest): ret = dict( channels=channels, ) - return json_response(ret) \ No newline at end of file + return json_response(ret) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 035dce17..a55cd1e5 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -171,7 +171,10 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic except APIError as e: raise ValidationError() from e - pathlib.Path(data.directory).mkdir(parents=True, exist_ok=True) + directory = pathlib.Path(data.directory) + directory = get_media_directory() / directory if not directory.is_absolute() else directory + directory.mkdir(parents=True, exist_ok=True) + channel = Channel() session.add(channel) session.flush([channel]) diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index 630feabf..f17a153a 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -316,7 +316,7 @@ async def test_channel_crud(test_session, async_client, test_directory, test_dow # Can't update channel that doesn't exist request, response = await async_client.put('/api/videos/channels/doesnt_exist', - content=json.dumps(new_channel)) + content=json.dumps(new_channel)) assert response.status_code == HTTPStatus.NOT_FOUND # Delete the new channel diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 3dd2ebd5..5131e43a 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -9,7 +9,6 @@ from uuid import uuid4 import mock import pytest from PIL import Image -from requests import session from modules.videos.downloader import VideoDownloader, ChannelDownloader from modules.videos.lib import set_test_channels_config, set_test_downloader_config, format_videos_destination diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 71c1dd3e..d6329cb7 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -3,6 +3,7 @@ import dataclasses import html import pathlib import re +from datetime import datetime from typing import Generator, Type from typing import Tuple, Optional @@ -38,13 +39,14 @@ class VideoInfoJSON: channel_source_id: str | None = None channel_url: str | None = None duration: int | None = None + epoch: int | None = None title: str | None = None + upload_date: datetime | None = None url: str | None = None view_count: int | None = None - epoch: int | None = None -def process_video_info_json(video: Video) -> VideoInfoJSON: +def extract_video_info_json(video: Video) -> VideoInfoJSON: """ Parse the Video's info json file, return the relevant data. """ @@ -53,12 +55,17 @@ def process_video_info_json(video: Video) -> VideoInfoJSON: title = info_json.get('fulltitle') or info_json.get('title') video_info_json.title = html.unescape(title) if title else None - video_info_json.duration = int(i) if (i := info_json.get('duration')) else None - video_info_json.view_count = int(i) if (i := info_json.get('view_count')) else None - video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None + upload_date = dates.strpdate(i) if (i := info_json.get('upload_date')) else None + if upload_date: + upload_date = upload_date.astimezone(pytz.UTC) + video_info_json.channel_source_id = info_json.get('channel_id') or info_json.get('uploader_id') or None video_info_json.channel_url = info_json.get('channel_url') or info_json.get('uploader_url') or None + video_info_json.duration = int(i) if (i := info_json.get('duration')) else None video_info_json.epoch = int(i) if (i := info_json.get('epoch')) else None + video_info_json.upload_date = upload_date + video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None + video_info_json.view_count = int(i) if (i := info_json.get('view_count')) else None return video_info_json @@ -76,15 +83,16 @@ def validate_video(video: Video, channel_generate_poster: bool): A Video is also valid when it has a JPEG poster, if any. If no poster can be found, it will be generated from the video file. """ - info_json_path = video.info_json_path - json_data_missing = bool(video.file_group.title) and bool(video.file_group.length) and bool(video.view_count) \ - and bool(video.file_group.url) and bool(video.file_group.download_datetime) - if info_json_path and json_data_missing is False: + json_published_datetime = None + + if video.info_json_path: # These properties can be found in the info json. - video_info_json = process_video_info_json(video) + video_info_json = extract_video_info_json(video) video.file_group.title = video_info_json.title video.file_group.length = video_info_json.duration video.file_group.url = video_info_json.url + json_published_datetime = video.file_group.published_datetime = \ + video_info_json.upload_date or video.file_group.published_datetime video.file_group.download_datetime = from_timestamp(video_info_json.epoch) if video_info_json.epoch else None # View count will probably be overwritten by more recent data when this Video's Channel is # updated. @@ -117,7 +125,8 @@ def validate_video(video: Video, channel_generate_poster: bool): if not published_date.tzinfo: published_date = published_date.astimezone(pytz.UTC) video.file_group.title = video.file_group.title or html.unescape(title) - video.file_group.published_datetime = published_date + # Trust info json upload_date before file name datetime. + video.file_group.published_datetime = json_published_datetime or published_date video.source_id = video.source_id or source_id if channel_generate_poster: diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index e558d596..96b84323 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -179,7 +179,7 @@ async def test_api_download_video_tags(test_session, async_client, tag_factory): @pytest.mark.asyncio -async def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, +async def test_search_videos_file(test_session, test_directory, video_with_search_factory, assert_video_search): """Test that videos can be searched and that their order is by their textsearch rank.""" # These captions have repeated letters, so they will be higher in the ranking. diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 8b2793eb..a2e40694 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -89,6 +89,25 @@ def test_validate_video(test_directory, video_factory, image_bytes_factory, vide assert vid1.source_id == '1234567890' assert vid1.file_group.title == 'The Title' + # Replace info json, FileGroup attributes should be replaced. + info_json = { + 'channel_id': 'channel id', + 'channel_url': 'https://example.com/example_video_json_channel', + 'duration': 635, + 'epoch': 123456789, + 'fulltitle': 'The full title', + 'id': 'some long id', + 'title': 'The title', + 'upload_date': '20240917', + 'uploader_id': 'uploader id', + 'uploader_url': 'https://example.com/example_video_json_uploader', + 'view_count': 406 + } + vid1.replace_info_json(info_json) + validate_video(vid1, False) + assert vid1.file_group.published_datetime and vid1.file_group.published_datetime.year == 2024 + assert vid1.file_group.title == 'The full title' + # A PNG is replaced. vid2 = video_factory(with_video_file=True, with_poster_ext='.png') vid2.poster_path.with_suffix('.jpg').write_bytes(image_bytes_factory()) # New poster exists, replace it. diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index 3bb5e5d0..c9ae4323 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -6,7 +6,7 @@ from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi.api_utils import json_response -from wrolpi.common import logger, wrol_mode_check, run_after +from wrolpi.common import logger, wrol_mode_check from wrolpi.errors import InvalidOrderBy, ValidationError from wrolpi.events import Events from wrolpi.schema import JSONErrorResponse diff --git a/modules/zim/models.py b/modules/zim/models.py index 5c06339b..7441977a 100644 --- a/modules/zim/models.py +++ b/modules/zim/models.py @@ -12,7 +12,7 @@ from sqlalchemy.orm.exc import NoResultFound # noqa from modules.zim.errors import UnknownZimEntry, UnknownZimTagEntry from wrolpi import dates, tags -from wrolpi.common import Base, extract_html_text, logger, get_relative_to_media_directory +from wrolpi.common import Base, logger, get_relative_to_media_directory from wrolpi.dates import TZDateTime from wrolpi.db import optional_session, get_db_curs from wrolpi.downloader import Download, download_manager diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index a92800cb..3555d035 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -250,7 +250,8 @@ async def test_zim_download(test_session, kiwix_download_zim, test_directory, te @pytest.mark.asyncio -async def test_zim_tag_migration(async_client, test_session, test_directory, zim_factory, tag_factory, test_tags_config): +async def test_zim_tag_migration(async_client, test_session, test_directory, zim_factory, tag_factory, + test_tags_config): """When an outdated Zim file is deleted, the Tags should be moved to the latest Zim.""" zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim. zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim') diff --git a/pytest.ini b/pytest.ini index 64d9ad94..1c418e98 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,4 @@ asyncio_mode = auto testpaths = wrolpi modules +asyncio_default_fixture_loop_scope = function \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 11f4266b..d574162d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,25 +1,25 @@ -Pillow==10.3.0 -Pint==0.21 -PyYAML==6.0.1 +Pillow==10.4.0 +Pint==0.24.3 +PyYAML==6.0.2 SQLAlchemy[asyncio]==1.3.22 -aiohttp~=3.9.5 +aiohttp==3.10.5 alembic==1.10.4 -beautifulsoup4==4.12.2 -cachetools==5.3.0 +beautifulsoup4==4.12.3 +cachetools==5.5.0 ebooklib==0.18 -feedparser==6.0.10 +feedparser==6.0.11 jc==1.25.3 -libzim==3.3.0.post0 +libzim==3.5.0 mock==5.1.0 -psutil==5.9.5 -psycopg2-binary==2.9.6 -pypdf==3.17.0 -pytest-asyncio==0.21.0 -pytest==7.3.1 -python-docx==1.1.0 -python-dotenv==1.0.0 +psutil==6.0.0 +psycopg2-binary==2.9.9 +pypdf==4.3.1 +pytest-asyncio==0.24.0 +pytest==8.3.3 +python-docx==1.1.2 +python-dotenv==1.0.1 python-magic~=0.4.27 -pytz==2023.3 +pytz==2024.2 sanic-ext==23.6.0 sanic-testing==22.6.0 # 23.6.0 has error with "expandtabs" sanic==23.6.0 # This is the latest version of Sanic that avoids this bug: https://github.com/sanic-org/sanic/issues/2921 @@ -27,5 +27,5 @@ selenium==4.9.1 srt==3.5.3 vininfo[cli]==1.8.0 websockets==12.0 -webvtt-py==0.4.6 +webvtt-py==0.5.1 yt-dlp==2024.8.6 \ No newline at end of file diff --git a/test/bad_caption.en.vtt b/test/bad_caption.en.vtt deleted file mode 100644 index 463b74ef..00000000 --- a/test/bad_caption.en.vtt +++ /dev/null @@ -1,8 +0,0 @@ -WEBVTT -Kind: captions -Language: en - -00:00:00.070 --> 00:00:06.380 -this caption is bad - -00:05:08.650 --> 00:05:11.26 diff --git a/wrolpi/__init__.py b/wrolpi/__init__.py index 339506f7..0bcb7fa7 100644 --- a/wrolpi/__init__.py +++ b/wrolpi/__init__.py @@ -1,6 +1,6 @@ import wrolpi.files.api # noqa Import from files so the blueprint is attached. import wrolpi.scrape_downloader # noqa -import wrolpi.status # noqa +import wrolpi.status # noqa BEFORE_STARTUP_FUNCTIONS = [] diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py index a535b82b..0ff68646 100644 --- a/wrolpi/cmd.py +++ b/wrolpi/cmd.py @@ -85,4 +85,4 @@ YT_DLP_BIN = which( ) FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe') FFMPEG_BIN = which('ffmpeg', '/usr/bin/ffmpeg') -JC_BIN = which('jc', '/usr/bin/jc') \ No newline at end of file +JC_BIN = which('jc', '/usr/bin/jc') diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 662c0e79..ebeeff3a 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -382,14 +382,6 @@ def srt_file3(test_directory) -> pathlib.Path: yield destination -@pytest.fixture -def bad_vtt_file(test_directory) -> pathlib.Path: - """Return a copy of the bad_caption VTT file in the `test_directory`.""" - destination = test_directory / f'{uuid4()}.en.vtt' - shutil.copy(PROJECT_DIR / 'test/bad_caption.en.vtt', destination) - yield destination - - @pytest.fixture def example_pdf(test_directory) -> pathlib.Path: destination = test_directory / 'pdf example.pdf' diff --git a/wrolpi/events.py b/wrolpi/events.py index 7a55ddd8..e4603271 100644 --- a/wrolpi/events.py +++ b/wrolpi/events.py @@ -1,5 +1,3 @@ -import multiprocessing - __all__ = ['Events', 'get_events'] from datetime import datetime diff --git a/wrolpi/tags.py b/wrolpi/tags.py index 208a1cdb..4416f70b 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -1,7 +1,6 @@ import contextlib import pathlib from datetime import datetime -from pprint import pprint from typing import List, Dict, Tuple, Optional from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger diff --git a/wrolpi/test/test_captions.py b/wrolpi/test/test_captions.py index f13919ba..a0d2fecb 100644 --- a/wrolpi/test/test_captions.py +++ b/wrolpi/test/test_captions.py @@ -1,15 +1,8 @@ from wrolpi import captions -def test_bad_caption(bad_vtt_file): - """A caption file that cannot be parsed is ignored.""" - result = captions.read_captions(bad_vtt_file) - assert result is None - - def test_get_caption_text(vtt_file1, srt_file3): expected = ['okay welcome to this session this is', - 'okay welcome to this session this is', 'okay welcome to this session this is\ncalled the kinetic bunny need to meet', 'called the kinetic bunny need to meet', 'called the kinetic bunny need to meet\nthe virtual bonnie emilio stop this is', diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index b10f6244..34527318 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -5,7 +5,6 @@ from datetime import datetime from http import HTTPStatus from itertools import zip_longest from unittest import mock -from unittest.mock import AsyncMock import pytest import pytz diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 16d2a6b3..03fb1c64 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -5,7 +5,6 @@ from itertools import zip_longest import pytest from mock import mock from sanic import Request -from selenium.webdriver.common.devtools.v112.runtime import await_promise from wrolpi.admin import HotspotStatus from wrolpi.api_utils import json_error_handler diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 3b86d4fe..fac3fc27 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -49,6 +49,7 @@ LOG_LEVEL_INT = logging.INFO if '-vv' in sys.argv: LOG_LEVEL_INT = logging.DEBUG elif '-vvv' in sys.argv: + # SQLAlchemy will increase logging. LOG_LEVEL_INT = logging.NOTSET LOG_LEVEL = logging.getLevelName(LOG_LEVEL_INT)