From 9ffe6ee3f66fbbd68fd331162bf5ab4cc36c7b0c Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 8 Dec 2022 13:45:46 -0700 Subject: [PATCH] Info JSON epoch is time of download, not time of upload. --- app/src/components/Archive.js | 6 ++--- app/src/components/Common.js | 12 ++++----- app/src/components/Files.js | 6 ++--- app/src/components/VideoPlayer.js | 4 +-- app/src/components/Videos.js | 4 +-- modules/videos/lib.py | 12 --------- modules/videos/test/test_lib.py | 41 ------------------------------- 7 files changed, 16 insertions(+), 69 deletions(-) diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 5967da41..65df5d80 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -29,7 +29,7 @@ import { SearchInput, TabLinks, textEllipsis, - uploadDate, + epochToDateString, useTitle } from "./Common"; import {deleteArchives, postDownload} from "../api"; @@ -151,7 +151,7 @@ function ArchivePage() {
{textEllipsis(archive.title || archive.url, 100)}
 -
{uploadDate(archive.archive_datetime)}
+
{epochToDateString(archive.archive_datetime)}
{domainHeader} {urlHeader} @@ -208,7 +208,7 @@ export function ArchiveCard({file}) { }

- {uploadDate(archive.archive_datetime)} + {epochToDateString(archive.archive_datetime)}

diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 9c16fe7e..bc70f895 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -106,14 +106,14 @@ export function Duration({video}) { } } -export function uploadDate(d) { +export function epochToDateString(date) { // Convert a date integer to a human-readable date format. - let upload_date = <>; - if (d) { - upload_date = new Date(d * 1000); - upload_date = `${upload_date.getFullYear()}-${upload_date.getMonth() + 1}-${upload_date.getDate()}`; + let d = <>; + if (date) { + d = new Date(date * 1000); + d = `${d.getFullYear()}-${d.getMonth() + 1}-${d.getDate()}`; } - return upload_date; + return d; } export function CardLink({to, newTab = false, ...props}) { diff --git a/app/src/components/Files.js b/app/src/components/Files.js index 123f24f7..ee75cb22 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -25,7 +25,7 @@ import { PageContainer, Paginator, textEllipsis, - uploadDate, + epochToDateString, useTitle } from "./Common"; import {useBrowseFiles, useQuery, useSearchFiles} from "../hooks/customHooks"; @@ -166,7 +166,7 @@ function ImageCard({file}) { -

{uploadDate(file.modified / 1000)}

+

{epochToDateString(file.modified / 1000)}

{humanFileSize(file.size)}

@@ -203,7 +203,7 @@ function FileCard({file}) { {textEllipsis(file.title || file.stem || file.path)} -

{uploadDate(file.modified / 1000)}

+

{epochToDateString(file.modified / 1000)}

{size}

diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index c09e39cc..0af3d942 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -2,7 +2,7 @@ import React, {useContext, useState} from 'react'; import {deleteVideos} from "../api"; import {Link, useNavigate, useParams} from "react-router-dom"; import _ from "lodash"; -import {BackButton, humanFileSize, humanNumber, PageContainer, uploadDate, useTitle} from "./Common"; +import {BackButton, epochToDateString, humanFileSize, humanNumber, PageContainer, useTitle} from "./Common"; import {Confirm} from "semantic-ui-react"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import Container from "semantic-ui-react/dist/commonjs/elements/Container"; @@ -175,7 +175,7 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
{video.title || video.video_path}
- {video.upload_date &&

{uploadDate(video.upload_date)}

} + {video.upload_date &&

{epochToDateString(video.upload_date)}

}

{channel && {channel.name} diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index fc41984c..a8316aa7 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -14,7 +14,7 @@ import { SearchInput, TabLinks, textEllipsis, - uploadDate, + epochToDateString, useTitle } from "./Common" import VideoPage from "./VideoPlayer"; @@ -310,7 +310,7 @@ export function VideoCard({file}) { const {s} = useContext(ThemeContext); let video_url = `/videos/video/${video.id}`; - const upload_date = uploadDate(video.upload_date); + const upload_date = epochToDateString(video.upload_date); // A video may not have a channel. const channel = video.channel ? video.channel : null; let channel_url = null; diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 514b6e43..4dc523a2 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -29,7 +29,6 @@ DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week class VideoInfoJSON(object): channel_source_id = None duration = None - epoch = None title = None url = None view_count = None @@ -49,14 +48,6 @@ def process_video_info_json(video: Video) -> VideoInfoJSON: video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None video_info_json.channel_source_id = info_json.get('channel_id') or None - # Get `epoch` from top of JSON. Convert to datetime. - if epoch := info_json.get('epoch'): - if '.' in str(epoch): - epoch = float(epoch) - else: - epoch = int(epoch) - video_info_json.epoch = from_timestamp(epoch) if epoch else None - return video_info_json @@ -89,8 +80,6 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session video.title = video_info_json.title video.duration = video_info_json.duration video.url = video_info_json.url - # Epoch is more precise than upload_date. - video.upload_date = video_info_json.epoch # View count will probably be overwritten by more recent data when this Video's Channel is # updated. video.view_count = video.view_count or video_info_json.view_count @@ -105,7 +94,6 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session # These are the least trusted, so anything already on the video should be trusted. _, upload_date, source_id, title = parse_video_file_name(video_path) video.title = video.title or html.unescape(title) - # Prefer epoch over upload_date from file. video.upload_date = video.upload_date or upload_date video.source_id = video.source_id or source_id diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 7439534f..3f777200 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -197,47 +197,6 @@ async def test_get_statistics(test_session, video_factory, channel_factory): assert 'historical' in result['statistics'] -@skip_circleci -@pytest.mark.asyncio -async def test_video_epoch(test_session, test_directory, video_file): - """The epoch in a Video's info_json is more precise and should be used when available. - - Fallback to the upload_date in the Video file's name. - """ - # The date from a video file's name is assumed to be midnight in UTC. - upload_date = '20221206' - - destination = test_directory / f'test channel_{upload_date}_the title.mp4' - video_file.rename(destination) - video_file = destination - - await refresh_files() - assert test_session.query(File).count() == 1 - assert test_session.query(Video).count() == 1 - - video: Video = test_session.query(Video).one() - # Upload date is converted to local time. - assert video.title == 'the title' - assert video.upload_date == strptime('2022-12-05 17:00:00'), \ - 'Video upload_date should be the date from the file name.' - assert not video.info_json_file, 'Video should not have info json file' - - # Write info_json with more precise epoch. - info_json_path = video_file.with_suffix('.info.json') - info_json = {'epoch': 1656783193} - info_json_path.write_text(json.dumps(info_json)) - # Invalidate video so the JSON will be processed again. - video.validated = False - test_session.commit() - - # Epoch should replace file upload_date. - await refresh_files() - assert test_session.query(File).count() == 2 - assert video.info_json_file, 'Video info json was not discovered.' - assert video.upload_date == strptime('2022-07-02 11:33:13'), 'Epoch from info_json did not replace file upload_date' - assert video_file.is_file() and info_json_path.is_file(), 'Files should contain the old date.' - - @pytest.mark.asyncio async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory): # A Video without associated files is not orphaned.