From 0315def174fc47ec4f61ed86c06d1b18b9b20c91 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 25 Jan 2022 08:51:55 -0700 Subject: [PATCH] Videos can be sorted by their modification date. --- ...97c1313f986_video_modification_datetime.py | 34 +++++++++++++++++++ app/src/components/Common.js | 29 ++++++++++------ app/src/components/VideoPlayer.js | 6 ++-- modules/videos/lib.py | 9 +++-- modules/videos/models.py | 2 ++ modules/videos/test/test_lib.py | 14 +++++++- modules/videos/video/lib.py | 4 +++ wrolpi/conftest.py | 1 + wrolpi/dates.py | 19 +++++++++++ 9 files changed, 101 insertions(+), 17 deletions(-) create mode 100644 alembic/versions/b97c1313f986_video_modification_datetime.py diff --git a/alembic/versions/b97c1313f986_video_modification_datetime.py b/alembic/versions/b97c1313f986_video_modification_datetime.py new file mode 100644 index 00000000..4939cdd7 --- /dev/null +++ b/alembic/versions/b97c1313f986_video_modification_datetime.py @@ -0,0 +1,34 @@ +"""Video modification_datetime + +Revision ID: b97c1313f986 +Revises: 3c33b1dae069 +Create Date: 2022-01-25 08:00:26.944183 + +""" +import os +from alembic import op +from sqlalchemy.orm import Session + + +# revision identifiers, used by Alembic. +revision = 'b97c1313f986' +down_revision = '3c33b1dae069' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE video ADD COLUMN modification_datetime TIMESTAMPTZ') + session.execute('CREATE INDEX video_modification_datetime_idx ON video USING BRIN(modification_datetime)') + session.execute('UPDATE video SET validated=false') # get modification time for every video duration validation. + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('DROP INDEX IF EXISTS video_modification_datetime_idx') + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS modification_datetime') diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 09ff7d3d..29877e52 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -166,7 +166,9 @@ export let videoOrders = [ {key: 'size', value: 'size', text: 'Smallest', title: 'Smallest Videos'}, {key: '-view_count', value: '-view_count', text: 'Most Views', title: 'Most Views'}, {key: 'view_count', value: 'view_count', text: 'Least Views', title: 'Least Views'}, -]; + {key: '-modification_datetime', value: '-modification_datetime', text: 'Newest File', title: 'Newest File'}, + {key: 'modification_datetime', value: 'modification_datetime', text: 'Oldest File', title: 'Oldest File'}, +] export let searchOrders = [ {key: 'rank', value: 'rank', text: 'Search Rank', title: 'Search Results'}, @@ -222,6 +224,11 @@ export function secondsToDate(seconds) { return `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`; } +export function secondsToTimestamp(seconds) { + let d = new Date(seconds * 1000); + return `${d.getFullYear()}-${d.getMonth() + 1}-${d.getDate()} ${d.getHours()}:${d.getMinutes()}:${d.getSeconds()}`; +} + export function humanFileSize(bytes, si = false, dp = 1) { const thresh = si ? 1000 : 1024; @@ -610,15 +617,15 @@ export function WROLModeMessage({content}) { } // Thanks https://www.npmjs.com/package/text-ellipsis -export function textEllipsis(str, maxLength, { side = "end", ellipsis = "..." } = {}) { - if (str.length > maxLength) { - switch (side) { - case "start": - return ellipsis + str.slice(-(maxLength - ellipsis.length)); - case "end": - default: - return str.slice(0, maxLength - ellipsis.length) + ellipsis; +export function textEllipsis(str, maxLength, {side = "end", ellipsis = "..."} = {}) { + if (str.length > maxLength) { + switch (side) { + case "start": + return ellipsis + str.slice(-(maxLength - ellipsis.length)); + case "end": + default: + return str.slice(0, maxLength - ellipsis.length) + ellipsis; + } } - } - return str; + return str; } diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index 2874c056..e2947804 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -3,7 +3,7 @@ import Icon from "semantic-ui-react/dist/commonjs/elements/Icon"; import {deleteVideo, favoriteVideo} from "../api"; import Button from "semantic-ui-react/dist/commonjs/elements/Button"; import {Link} from "react-router-dom"; -import {humanFileSize, humanNumber, uploadDate, VideoCard} from "./Common"; +import {humanFileSize, humanNumber, secondsToTimestamp, uploadDate, VideoCard} from "./Common"; import {Confirm, Container, Segment, Tab} from "semantic-ui-react"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; @@ -92,8 +92,8 @@ function VideoPage(props) {

File Name

{video.video_path}
-

Source ID

-

{video.source_id || 'N/A'}

+

File Modification Time

+

{video.modification_datetime ? secondsToTimestamp(video.modification_datetime) : null}

} diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 08439353..053bf500 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -10,6 +10,7 @@ from sqlalchemy.orm import Session from wrolpi.common import logger, chunks, get_config from wrolpi.common import save_settings_config +from wrolpi.dates import from_timestamp from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST @@ -188,8 +189,12 @@ def validate_video(video: Video, channel_generate_poster: bool): video.duration = get_video_duration(video_path) if not video.caption and video.caption_path: video.caption = get_video_captions(video) - if not video.size: - video.size = video_path.stat().st_size + + if not video.size or not video.modification_datetime: + stat = video_path.stat() + video.size = video.size or stat.st_size + video.modification_datetime = video.modification_datetime or from_timestamp(stat.st_mtime) + if not video.poster_path: # Video poster is not found, lets check near the video file. for ext in ('.jpg', '.jpeg', '.webp', '.png'): diff --git a/modules/videos/models.py b/modules/videos/models.py index c6a52eeb..298fb1a0 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -38,6 +38,7 @@ class Video(ModelHelper, Base): censored = Column(Boolean, default=False) duration = Column(Integer) favorite = Column(TZDateTime) + modification_datetime = Column(TZDateTime) size = Column(Integer) source_id = Column(String) title = Column(String) @@ -237,6 +238,7 @@ class Video(ModelHelper, Base): caption_path=self.caption_path, channel=self.channel, channel_id=self.channel_id, + modification_datetime=self.modification_datetime, duration=self.duration, favorite=self.favorite, id=self.id, diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index dac795ce..b16a66f5 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -9,7 +9,7 @@ import sqlalchemy from modules.videos.channel import lib from modules.videos.common import apply_info_json -from modules.videos.lib import validate_videos, parse_video_file_name +from modules.videos.lib import validate_videos, parse_video_file_name, upsert_video from modules.videos.models import Channel, Video from modules.videos.video.lib import video_search from wrolpi.dates import local_timezone @@ -267,3 +267,15 @@ def test_validate_does_not_generate_when_disabled(test_session, channel_factory, assert not vid1.poster_path assert vid2.poster_path assert str(vid3.poster_path).endswith('webp') + + +def test_video_modification_datetime(test_session, simple_channel, video_file): + """ + A Video has a modification_datetime which represents the video file's mtime. + """ + expected = local_timezone(datetime(2000, 1, 1, 0, 0, 0)) + with mock.patch('modules.videos.lib.from_timestamp') as mock_from_timestamp: + mock_from_timestamp.return_value = expected + video = upsert_video(test_session, video_file, simple_channel) + test_session.commit() + assert video.modification_datetime == expected diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index dfa13fdd..274e5184 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -55,6 +55,8 @@ VIDEO_ORDERS = { '-viewed': 'viewed DESC', 'view_count': 'view_count ASC', '-view_count': 'view_count DESC', + 'modification_datetime': 'modification_datetime ASC', + '-modification_datetime': 'modification_datetime DESC', } NO_NULL_ORDERS = { 'viewed': '\nAND viewed IS NOT NULL', @@ -65,6 +67,8 @@ NO_NULL_ORDERS = { '-size': '\nAND size IS NOT NULL', 'view_count': '\nAND view_count IS NOT NULL', '-view_count': '\nAND view_count IS NOT NULL', + 'modification_datetime': '\nAND modification_datetime IS NOT NULL', + '-modification_datetime': '\nAND modification_datetime IS NOT NULL', } DEFAULT_VIDEO_ORDER = 'rank' VIDEO_QUERY_LIMIT = 20 diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 8ae1bb6b..fede0150 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -16,6 +16,7 @@ from sqlalchemy.engine import Engine, create_engine from sqlalchemy.orm import Session, sessionmaker from wrolpi.common import set_test_media_directory, Base, get_example_config +from wrolpi.dates import fake_now # noqa from wrolpi.db import postgres_engine, get_db_args from wrolpi.downloader import DownloadManager from wrolpi.root_api import BLUEPRINTS, api_app diff --git a/wrolpi/dates.py b/wrolpi/dates.py index f36871fd..09dc36f6 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -1,6 +1,7 @@ import logging from datetime import datetime, date, timezone +import pytest import pytz from sqlalchemy import types @@ -10,6 +11,18 @@ from wrolpi.vars import DEFAULT_TIMEZONE_STR, DATETIME_FORMAT, DATETIME_FORMAT_M logger = logging.getLogger(__name__) DEFAULT_TIMEZONE = pytz.timezone(DEFAULT_TIMEZONE_STR) +TEST_DATETIME: datetime = None + + +@pytest.fixture +def fake_now(): + def _(dt: datetime): + if not dt.tzinfo: + dt = local_timezone(dt) + global TEST_DATETIME + TEST_DATETIME = dt + + return _ def set_timezone(tz: pytz.timezone): @@ -39,6 +52,8 @@ def now(tz: pytz.timezone = None) -> datetime: """ Get the current DateTime in the provided timezone. Timezone aware. """ + if TEST_DATETIME: + return TEST_DATETIME tz = tz or DEFAULT_TIMEZONE return datetime.utcnow().astimezone(tz) @@ -73,6 +88,10 @@ def strptime_ms(dt: str) -> datetime: return local_timezone(datetime.strptime(dt, DATETIME_FORMAT_MS)) +def from_timestamp(timestamp: float) -> datetime: + return local_timezone(datetime.fromtimestamp(timestamp)) + + class TZDateTime(types.TypeDecorator): impl = types.DateTime cache_ok = True