Info JSON epoch is time of download, not time of upload.

This commit is contained in:
Roland 2022-12-08 13:45:46 -07:00
parent ad405d30cd
commit 9ffe6ee3f6
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
7 changed files with 16 additions and 69 deletions

View file

@ -29,7 +29,7 @@ import {
SearchInput,
TabLinks,
textEllipsis,
uploadDate,
epochToDateString,
useTitle
} from "./Common";
import {deleteArchives, postDownload} from "../api";
@ -151,7 +151,7 @@ function ArchivePage() {
<ExternalCardLink to={singlefileUrl}>
<Header as='h2'>{textEllipsis(archive.title || archive.url, 100)}</Header>
</ExternalCardLink>
<Header as='h3'>{uploadDate(archive.archive_datetime)}</Header>
<Header as='h3'>{epochToDateString(archive.archive_datetime)}</Header>
{domainHeader}
{urlHeader}
@ -208,7 +208,7 @@ export function ArchiveCard({file}) {
</CardLink>}
<CardMeta {...s}>
<p>
{uploadDate(archive.archive_datetime)}
{epochToDateString(archive.archive_datetime)}
</p>
</CardMeta>
<CardDescription>

View file

@ -106,14 +106,14 @@ export function Duration({video}) {
}
}
export function uploadDate(d) {
export function epochToDateString(date) {
// Convert a date integer to a human-readable date format.
let upload_date = <></>;
if (d) {
upload_date = new Date(d * 1000);
upload_date = `${upload_date.getFullYear()}-${upload_date.getMonth() + 1}-${upload_date.getDate()}`;
let d = <></>;
if (date) {
d = new Date(date * 1000);
d = `${d.getFullYear()}-${d.getMonth() + 1}-${d.getDate()}`;
}
return upload_date;
return d;
}
export function CardLink({to, newTab = false, ...props}) {

View file

@ -25,7 +25,7 @@ import {
PageContainer,
Paginator,
textEllipsis,
uploadDate,
epochToDateString,
useTitle
} from "./Common";
import {useBrowseFiles, useQuery, useSearchFiles} from "../hooks/customHooks";
@ -166,7 +166,7 @@ function ImageCard({file}) {
</ExternalCardLink>
</CardHeader>
<CardMeta {...s}>
<p>{uploadDate(file.modified / 1000)}</p>
<p>{epochToDateString(file.modified / 1000)}</p>
</CardMeta>
<CardDescription {...s}>
<p>{humanFileSize(file.size)}</p>
@ -203,7 +203,7 @@ function FileCard({file}) {
{textEllipsis(file.title || file.stem || file.path)}
</ExternalCardLink>
</CardHeader>
<p>{uploadDate(file.modified / 1000)}</p>
<p>{epochToDateString(file.modified / 1000)}</p>
<p>{size}</p>
</CardContent>
</Card>

View file

@ -2,7 +2,7 @@ import React, {useContext, useState} from 'react';
import {deleteVideos} from "../api";
import {Link, useNavigate, useParams} from "react-router-dom";
import _ from "lodash";
import {BackButton, humanFileSize, humanNumber, PageContainer, uploadDate, useTitle} from "./Common";
import {BackButton, epochToDateString, humanFileSize, humanNumber, PageContainer, useTitle} from "./Common";
import {Confirm} from "semantic-ui-react";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Container from "semantic-ui-react/dist/commonjs/elements/Container";
@ -175,7 +175,7 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
<Segment>
<Header as='h2'>{video.title || video.video_path}</Header>
{video.upload_date && <h3>{uploadDate(video.upload_date)}</h3>}
{video.upload_date && <h3>{epochToDateString(video.upload_date)}</h3>}
<h3>
{channel && <Link to={`/videos/channel/${channel.id}/video`}>
{channel.name}

View file

@ -14,7 +14,7 @@ import {
SearchInput,
TabLinks,
textEllipsis,
uploadDate,
epochToDateString,
useTitle
} from "./Common"
import VideoPage from "./VideoPlayer";
@ -310,7 +310,7 @@ export function VideoCard({file}) {
const {s} = useContext(ThemeContext);
let video_url = `/videos/video/${video.id}`;
const upload_date = uploadDate(video.upload_date);
const upload_date = epochToDateString(video.upload_date);
// A video may not have a channel.
const channel = video.channel ? video.channel : null;
let channel_url = null;

View file

@ -29,7 +29,6 @@ DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week
class VideoInfoJSON(object):
channel_source_id = None
duration = None
epoch = None
title = None
url = None
view_count = None
@ -49,14 +48,6 @@ def process_video_info_json(video: Video) -> VideoInfoJSON:
video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None
video_info_json.channel_source_id = info_json.get('channel_id') or None
# Get `epoch` from top of JSON. Convert to datetime.
if epoch := info_json.get('epoch'):
if '.' in str(epoch):
epoch = float(epoch)
else:
epoch = int(epoch)
video_info_json.epoch = from_timestamp(epoch) if epoch else None
return video_info_json
@ -89,8 +80,6 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
video.title = video_info_json.title
video.duration = video_info_json.duration
video.url = video_info_json.url
# Epoch is more precise than upload_date.
video.upload_date = video_info_json.epoch
# View count will probably be overwritten by more recent data when this Video's Channel is
# updated.
video.view_count = video.view_count or video_info_json.view_count
@ -105,7 +94,6 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
# These are the least trusted, so anything already on the video should be trusted.
_, upload_date, source_id, title = parse_video_file_name(video_path)
video.title = video.title or html.unescape(title)
# Prefer epoch over upload_date from file.
video.upload_date = video.upload_date or upload_date
video.source_id = video.source_id or source_id

View file

@ -197,47 +197,6 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
assert 'historical' in result['statistics']
@skip_circleci
@pytest.mark.asyncio
async def test_video_epoch(test_session, test_directory, video_file):
"""The epoch in a Video's info_json is more precise and should be used when available.
Fallback to the upload_date in the Video file's name.
"""
# The date from a video file's name is assumed to be midnight in UTC.
upload_date = '20221206'
destination = test_directory / f'test channel_{upload_date}_the title.mp4'
video_file.rename(destination)
video_file = destination
await refresh_files()
assert test_session.query(File).count() == 1
assert test_session.query(Video).count() == 1
video: Video = test_session.query(Video).one()
# Upload date is converted to local time.
assert video.title == 'the title'
assert video.upload_date == strptime('2022-12-05 17:00:00'), \
'Video upload_date should be the date from the file name.'
assert not video.info_json_file, 'Video should not have info json file'
# Write info_json with more precise epoch.
info_json_path = video_file.with_suffix('.info.json')
info_json = {'epoch': 1656783193}
info_json_path.write_text(json.dumps(info_json))
# Invalidate video so the JSON will be processed again.
video.validated = False
test_session.commit()
# Epoch should replace file upload_date.
await refresh_files()
assert test_session.query(File).count() == 2
assert video.info_json_file, 'Video info json was not discovered.'
assert video.upload_date == strptime('2022-07-02 11:33:13'), 'Epoch from info_json did not replace file upload_date'
assert video_file.is_file() and info_json_path.is_file(), 'Files should contain the old date.'
@pytest.mark.asyncio
async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory):
# A Video without associated files is not orphaned.