Info JSON epoch is time of download, not time of upload.

This commit is contained in:
Roland 2022-12-08 13:45:46 -07:00
parent ad405d30cd
commit 9ffe6ee3f6
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
7 changed files with 16 additions and 69 deletions

View file

@ -29,7 +29,6 @@ DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week
class VideoInfoJSON(object):
channel_source_id = None
duration = None
epoch = None
title = None
url = None
view_count = None
@ -49,14 +48,6 @@ def process_video_info_json(video: Video) -> VideoInfoJSON:
video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None
video_info_json.channel_source_id = info_json.get('channel_id') or None
# Get `epoch` from top of JSON. Convert to datetime.
if epoch := info_json.get('epoch'):
if '.' in str(epoch):
epoch = float(epoch)
else:
epoch = int(epoch)
video_info_json.epoch = from_timestamp(epoch) if epoch else None
return video_info_json
@ -89,8 +80,6 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
video.title = video_info_json.title
video.duration = video_info_json.duration
video.url = video_info_json.url
# Epoch is more precise than upload_date.
video.upload_date = video_info_json.epoch
# View count will probably be overwritten by more recent data when this Video's Channel is
# updated.
video.view_count = video.view_count or video_info_json.view_count
@ -105,7 +94,6 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
# These are the least trusted, so anything already on the video should be trusted.
_, upload_date, source_id, title = parse_video_file_name(video_path)
video.title = video.title or html.unescape(title)
# Prefer epoch over upload_date from file.
video.upload_date = video.upload_date or upload_date
video.source_id = video.source_id or source_id

View file

@ -197,47 +197,6 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
assert 'historical' in result['statistics']
@skip_circleci
@pytest.mark.asyncio
async def test_video_epoch(test_session, test_directory, video_file):
"""The epoch in a Video's info_json is more precise and should be used when available.
Fallback to the upload_date in the Video file's name.
"""
# The date from a video file's name is assumed to be midnight in UTC.
upload_date = '20221206'
destination = test_directory / f'test channel_{upload_date}_the title.mp4'
video_file.rename(destination)
video_file = destination
await refresh_files()
assert test_session.query(File).count() == 1
assert test_session.query(Video).count() == 1
video: Video = test_session.query(Video).one()
# Upload date is converted to local time.
assert video.title == 'the title'
assert video.upload_date == strptime('2022-12-05 17:00:00'), \
'Video upload_date should be the date from the file name.'
assert not video.info_json_file, 'Video should not have info json file'
# Write info_json with more precise epoch.
info_json_path = video_file.with_suffix('.info.json')
info_json = {'epoch': 1656783193}
info_json_path.write_text(json.dumps(info_json))
# Invalidate video so the JSON will be processed again.
video.validated = False
test_session.commit()
# Epoch should replace file upload_date.
await refresh_files()
assert test_session.query(File).count() == 2
assert video.info_json_file, 'Video info json was not discovered.'
assert video.upload_date == strptime('2022-07-02 11:33:13'), 'Epoch from info_json did not replace file upload_date'
assert video_file.is_file() and info_json_path.is_file(), 'Files should contain the old date.'
@pytest.mark.asyncio
async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory):
# A Video without associated files is not orphaned.