Merge pull request #86 from lrnselfreliance/feature/download-date

Videos will have a modification date

Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
lrnselfreliance 2022-01-25 17:25:45 -07:00 • committed by Roland
commit 438feb363a
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
10 changed files with 121 additions and 29 deletions

View file

@ -0,0 +1,34 @@
"""Video modification_datetime
Revision ID: b97c1313f986
Revises: 3c33b1dae069
Create Date: 2022-01-25 08:00:26.944183
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = 'b97c1313f986'
down_revision = '3c33b1dae069'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE video ADD COLUMN modification_datetime TIMESTAMPTZ')
session.execute('CREATE INDEX video_modification_datetime_idx ON video USING BRIN(modification_datetime)')
session.execute('UPDATE video SET validated=false') # get modification time for every video duration validation.
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP INDEX IF EXISTS video_modification_datetime_idx')
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS modification_datetime')

View file

@ -166,7 +166,9 @@ export let videoOrders = [
{key: 'size', value: 'size', text: 'Smallest', title: 'Smallest Videos'},
{key: '-view_count', value: '-view_count', text: 'Most Views', title: 'Most Views'},
{key: 'view_count', value: 'view_count', text: 'Least Views', title: 'Least Views'},
];
{key: '-modification_datetime', value: '-modification_datetime', text: 'Newest File', title: 'Newest File'},
{key: 'modification_datetime', value: 'modification_datetime', text: 'Oldest File', title: 'Oldest File'},
]
export let searchOrders = [
{key: 'rank', value: 'rank', text: 'Search Rank', title: 'Search Results'},
@ -222,6 +224,11 @@ export function secondsToDate(seconds) {
return `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`;
}
export function secondsToTimestamp(seconds) {
let d = new Date(seconds * 1000);
return `${d.getFullYear()}-${d.getMonth() + 1}-${d.getDate()} ${d.getHours()}:${d.getMinutes()}:${d.getSeconds()}`;
}
export function humanFileSize(bytes, si = false, dp = 1) {
const thresh = si ? 1000 : 1024;
@ -610,15 +617,15 @@ export function WROLModeMessage({content}) {
}
// Thanks https://www.npmjs.com/package/text-ellipsis
export function textEllipsis(str, maxLength, { side = "end", ellipsis = "..." } = {}) {
if (str.length > maxLength) {
switch (side) {
case "start":
return ellipsis + str.slice(-(maxLength - ellipsis.length));
case "end":
default:
return str.slice(0, maxLength - ellipsis.length) + ellipsis;
export function textEllipsis(str, maxLength, {side = "end", ellipsis = "..."} = {}) {
if (str.length > maxLength) {
switch (side) {
case "start":
return ellipsis + str.slice(-(maxLength - ellipsis.length));
case "end":
default:
return str.slice(0, maxLength - ellipsis.length) + ellipsis;
}
}
}
return str;
return str;
}

View file

@ -3,7 +3,7 @@ import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {deleteVideo, favoriteVideo} from "../api";
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
import {Link} from "react-router-dom";
import {humanFileSize, humanNumber, uploadDate, VideoCard} from "./Common";
import {humanFileSize, humanNumber, secondsToTimestamp, uploadDate, VideoCard} from "./Common";
import {Confirm, Container, Segment, Tab} from "semantic-ui-react";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
@ -92,12 +92,18 @@ function VideoPage(props) {
<h3>File Name</h3>
<pre style={{backgroundColor: '#ccc', padding: '0.4em'}}>{video.video_path}</pre>
<h3>Source ID</h3>
<p>{video.source_id || 'N/A'}</p>
<h3>File Modification Time</h3>
<p>{video.modification_datetime ? secondsToTimestamp(video.modification_datetime) : null}</p>
</Tab.Pane>
}
let tabPanes = [descriptionPane, statisticsPane];
let captionsPane = {
menuItem: 'Captions', render: () => <Tab.Pane>
<pre>{video.caption}</pre>
</Tab.Pane>
};
let tabPanes = [descriptionPane, statisticsPane, captionsPane];
return (
<Container textAlign='left'>

View file

@ -75,16 +75,6 @@ yarn --network-timeout 10000
yarn run build
yarn global add serve
# Configure PostgreSQL
sudo -u postgres psql -c '\l' | grep wrolpi || (
sudo -u postgres createuser wrolpi &&
sudo -u postgres psql -c "alter user postgres password 'wrolpi'" &&
sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" &&
sudo -u postgres createdb -O wrolpi wrolpi
)
# Initialize the WROLPi database.
(cd /opt/wrolpi && /usr/bin/python3 /opt/wrolpi/main.py db upgrade)
# Install the WROLPi nginx config over the default nginx config.
cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf
cp /opt/wrolpi/50x.html /var/www/50x.html
@ -109,6 +99,16 @@ systemctl enable wrolpi-app.service
systemctl stop wrolpi-api.service
systemctl stop wrolpi-app.service
# Configure Postgresql. Do this after the API is stopped.
sudo -u postgres psql -c '\l' | grep wrolpi || (
sudo -u postgres createuser wrolpi &&
sudo -u postgres psql -c "alter user postgres password 'wrolpi'" &&
sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" &&
sudo -u postgres createdb -O wrolpi wrolpi
)
# Init/upgrade the WROLPi database.
(cd /opt/wrolpi && /usr/bin/python3 /opt/wrolpi/main.py db upgrade)
# Configure the hotspot.
bash /opt/wrolpi/scripts/hotspot.sh

View file

@ -10,6 +10,7 @@ from sqlalchemy.orm import Session
from wrolpi.common import logger, chunks, get_config
from wrolpi.common import save_settings_config
from wrolpi.dates import from_timestamp
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.media_path import MediaPath
from wrolpi.vars import PYTEST
@ -188,8 +189,12 @@ def validate_video(video: Video, channel_generate_poster: bool):
video.duration = get_video_duration(video_path)
if not video.caption and video.caption_path:
video.caption = get_video_captions(video)
if not video.size:
video.size = video_path.stat().st_size
if not video.size or not video.modification_datetime:
stat = video_path.stat()
video.size = video.size or stat.st_size
video.modification_datetime = video.modification_datetime or from_timestamp(stat.st_mtime)
if not video.poster_path:
# Video poster is not found, lets check near the video file.
for ext in ('.jpg', '.jpeg', '.webp', '.png'):

View file

@ -38,6 +38,7 @@ class Video(ModelHelper, Base):
censored = Column(Boolean, default=False)
duration = Column(Integer)
favorite = Column(TZDateTime)
modification_datetime = Column(TZDateTime)
size = Column(Integer)
source_id = Column(String)
title = Column(String)
@ -115,7 +116,7 @@ class Video(ModelHelper, Base):
"""
Add this video to it's Channel's skip list.
"""
if self.channel:
if self.channel and self.source_id:
self.channel.add_video_to_skip_list(self.source_id)
def set_favorite(self, favorite: bool) -> Optional[datetime]:
@ -237,6 +238,7 @@ class Video(ModelHelper, Base):
caption_path=self.caption_path,
channel=self.channel,
channel_id=self.channel_id,
modification_datetime=self.modification_datetime,
duration=self.duration,
favorite=self.favorite,
id=self.id,

View file

@ -9,7 +9,7 @@ import sqlalchemy
from modules.videos.channel import lib
from modules.videos.common import apply_info_json
from modules.videos.lib import validate_videos, parse_video_file_name
from modules.videos.lib import validate_videos, parse_video_file_name, upsert_video
from modules.videos.models import Channel, Video
from modules.videos.video.lib import video_search
from wrolpi.dates import local_timezone
@ -267,3 +267,15 @@ def test_validate_does_not_generate_when_disabled(test_session, channel_factory,
assert not vid1.poster_path
assert vid2.poster_path
assert str(vid3.poster_path).endswith('webp')
def test_video_modification_datetime(test_session, simple_channel, video_file):
"""
A Video has a modification_datetime which represents the video file's mtime.
"""
expected = local_timezone(datetime(2000, 1, 1, 0, 0, 0))
with mock.patch('modules.videos.lib.from_timestamp') as mock_from_timestamp:
mock_from_timestamp.return_value = expected
video = upsert_video(test_session, video_file, simple_channel)
test_session.commit()
assert video.modification_datetime == expected

View file

@ -31,7 +31,9 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
video.set_viewed()
previous_video, next_video = video.get_surrounding_videos()
caption = video.caption
video = video.__json__()
video['caption'] = caption
previous_video = previous_video.__json__() if previous_video else None
next_video = next_video.__json__() if next_video else None
@ -55,6 +57,8 @@ VIDEO_ORDERS = {
'-viewed': 'viewed DESC',
'view_count': 'view_count ASC',
'-view_count': 'view_count DESC',
'modification_datetime': 'modification_datetime ASC',
'-modification_datetime': 'modification_datetime DESC',
}
NO_NULL_ORDERS = {
'viewed': '\nAND viewed IS NOT NULL',
@ -65,6 +69,8 @@ NO_NULL_ORDERS = {
'-size': '\nAND size IS NOT NULL',
'view_count': '\nAND view_count IS NOT NULL',
'-view_count': '\nAND view_count IS NOT NULL',
'modification_datetime': '\nAND modification_datetime IS NOT NULL',
'-modification_datetime': '\nAND modification_datetime IS NOT NULL',
}
DEFAULT_VIDEO_ORDER = 'rank'
VIDEO_QUERY_LIMIT = 20

View file

@ -16,6 +16,7 @@ from sqlalchemy.engine import Engine, create_engine
from sqlalchemy.orm import Session, sessionmaker
from wrolpi.common import set_test_media_directory, Base, get_example_config
from wrolpi.dates import fake_now # noqa
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager
from wrolpi.root_api import BLUEPRINTS, api_app

View file

@ -1,6 +1,7 @@
import logging
from datetime import datetime, date, timezone
import pytest
import pytz
from sqlalchemy import types
@ -10,6 +11,18 @@ from wrolpi.vars import DEFAULT_TIMEZONE_STR, DATETIME_FORMAT, DATETIME_FORMAT_M
logger = logging.getLogger(__name__)
DEFAULT_TIMEZONE = pytz.timezone(DEFAULT_TIMEZONE_STR)
TEST_DATETIME: datetime = None
@pytest.fixture
def fake_now():
def _(dt: datetime):
if not dt.tzinfo:
dt = local_timezone(dt)
global TEST_DATETIME
TEST_DATETIME = dt
return _
def set_timezone(tz: pytz.timezone):
@ -39,6 +52,8 @@ def now(tz: pytz.timezone = None) -> datetime:
"""
Get the current DateTime in the provided timezone. Timezone aware.
"""
if TEST_DATETIME:
return TEST_DATETIME
tz = tz or DEFAULT_TIMEZONE
return datetime.utcnow().astimezone(tz)
@ -73,6 +88,10 @@ def strptime_ms(dt: str) -> datetime:
return local_timezone(datetime.strptime(dt, DATETIME_FORMAT_MS))
def from_timestamp(timestamp: float) -> datetime:
return local_timezone(datetime.fromtimestamp(timestamp))
class TZDateTime(types.TypeDecorator):
impl = types.DateTime
cache_ok = True