FileBrowser files can be tagged. A Tag icon is displayed on each file row in the browser.

This commit is contained in:
Roland 2023-04-01 11:52:41 -06:00
parent 655e944960
commit ae2b6daa33
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
29 changed files with 356 additions and 219 deletions

View file

@ -179,6 +179,8 @@ async def archive_modeler():
# Even if indexing fails, we mark it as indexed. We won't retry indexing this.
file_group.indexed = True
session.commit()
if processed < 20:
# Did not reach limit, do not query again.
break

View file

@ -45,6 +45,8 @@ async def video_modeler():
file_group.indexed = True
session.commit()
if processed < VIDEO_PROCESSING_LIMIT:
# Did not reach limit, do not query again.
break
@ -54,6 +56,7 @@ async def video_modeler():
# Sleep to catch cancel.
await asyncio.sleep(0)
@register_refresh_cleanup
@limit_concurrent(1)
def video_cleanup():

View file

@ -26,16 +26,6 @@ add_blueprint(bp)
logger = logger.getChild(__name__)
@content_bp.post('/download/<channel_id:int>')
@openapi.description('Update a channel catalog, download any missing videos')
@openapi.response(HTTPStatus.OK, schema.StreamResponse)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@wrol_mode_check
def download(_, channel_id: int = None):
channel_lib.download_channel(channel_id)
return response.empty()
@content_bp.post('/tag')
@openapi.definition(
description='Associate a Video with a Tag',

View file

@ -215,7 +215,7 @@ def check_for_video_corruption(video_path: Path) -> bool:
return corrupt
def update_view_counts(channel_id: int):
async def update_view_counts(channel_id: int):
"""Update view_count for all Videos in a channel using its info_json file."""
with get_db_session() as session:
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()

View file

@ -14,7 +14,7 @@ from yt_dlp.extractor import YoutubeTabIE # noqa
from yt_dlp.utils import UnsupportedError, DownloadError
from wrolpi.cmd import YT_DLP_BIN
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.db import optional_session
@ -438,7 +438,7 @@ def update_channel_catalog(channel: Channel, info: dict):
It is expected that any missing videos will be downloaded later.
"""
logger.info(f'Downloading video list for {channel.name} at {channel.url} This may take several minutes.')
logger.info(f'Downloading video list for {channel} at {channel.url}')
# Resolve all entries to dictionaries.
entries = info['entries'] = list(info['entries'])
@ -452,17 +452,8 @@ def update_channel_catalog(channel: Channel, info: dict):
if entry['title'] == 'Uploads':
logger.info('Youtube-DL gave back a list of URLs, found the "Uploads" URL and using it.')
info = extract_info(entry['url'])
entries = info['entries'] = list(info['entries'])
break
# This is all the source id's that are currently available.
try:
all_source_ids = {i['id'] for i in entries}
except KeyError as e:
logger.warning(f'No ids for entries! Was the channel update successful? Is the channel URL correct?')
logger.warning(f'entries: {entries}')
raise KeyError('No id key for entry!') from e
with get_db_session(commit=True) as session:
# Get the channel in this new context.
channel: Channel = session.query(Channel).filter_by(id=channel.id).one()
@ -478,8 +469,10 @@ def update_channel_catalog(channel: Channel, info: dict):
with info_json_path.open('wt') as fh:
json.dump(info, fh, indent=2)
logger.info(f'Finished downloading video list for {channel} found {len(entries)} videos')
# Update all view counts using the latest from the Channel's info_json.
update_view_counts(channel_id)
background_task(update_view_counts(channel_id))
def _find_all_missing_videos(channel_id: id) -> List[Tuple]:
@ -552,8 +545,11 @@ async def find_all_missing_videos(channel_id: int = None):
logger.warning(f'Video {channel.name} / {source_id} is not in {channel.name} info_json')
continue
if not match_regex or (match_regex and missing_video['title'] and match_regex.match(missing_video['title'])):
# No title match regex, or the title matches the regex.
if not match_regex:
# No title match regex, yield all videos.
yield video_id, source_id, missing_video
if match_regex and missing_video['title'] and match_regex.match(missing_video['title']):
# Title matches the regex.
yield video_id, source_id, missing_video

View file

@ -95,6 +95,8 @@ def validate_video(video: Video, channel_generate_poster: bool):
video.channel_id = get_channel_id_by_source_id(source_id=video_info_json.channel_source_id)
video_path = video.video_path
if not video_path:
logger.error(f'{video} does not have video_path!')
if video_path and (not video.file_group.title or not video.upload_date or not video.source_id):
# Video is missing things that can be extracted from the video file name.

View file

@ -43,7 +43,7 @@ class Video(ModelHelper, Base):
v = None
if self.video_path:
v = self.video_path.relative_to(get_media_directory())
return f'<Video id={self.id} title={repr(self.file_group.name)} path={v} channel={self.channel_id} ' \
return f'<Video id={self.id} title={repr(self.file_group.title)} path={v} channel={self.channel_id} ' \
f'source_id={repr(self.source_id)}>'
def __json__(self) -> dict:
@ -60,6 +60,7 @@ class Video(ModelHelper, Base):
channel=channel,
channel_id=self.channel_id,
description=self.file_group.c_text or self.get_video_description(),
duration=self.duration,
id=self.id,
info_json_file=self.info_json_file,
info_json_path=self.info_json_path,
@ -446,18 +447,18 @@ class Channel(ModelHelper, Base):
self_id = self.id
# Refresh all files within this channel's directory first.
await refresh_files(self.directory, send_events=send_events)
await refresh_files([self.directory], send_events=send_events)
# Apply any info_json (update view counts) second.
from modules.videos.common import update_view_counts
if PYTEST:
update_view_counts(self_id)
await update_view_counts(self_id)
self.refreshed = True
else:
# Perform info_json in background task. Channel will be marked as refreshed after this completes.
async def _():
update_view_counts(self_id)
await update_view_counts(self_id)
with get_db_session(commit=True) as session:
channel: Channel = session.query(Channel).filter(Channel.id == self_id).one()
channel.refreshed = True

View file

@ -149,15 +149,15 @@ def test_channels_with_videos(test_session, test_client, test_directory, channel
def test_api_download_channel(test_session, test_client, simple_channel):
"""A Channel download (a catalog update) can be triggered via the API."""
# simple_channel does not have a download record.
request, response = test_client.post(f'/api/videos/download/{simple_channel.id}')
request, response = test_client.post(f'/api/videos/channels/download/{simple_channel.id}')
assert response.status_code == HTTPStatus.BAD_REQUEST, response.json
assert 'not have a download' in response.json['message']
# Add a download frequency to the channel, this should also create a download.
simple_channel.update(dict(download_frequency=DownloadFrequency.daily))
test_session.commit()
request, response = test_client.post(f'/api/videos/download/{simple_channel.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = test_client.post(f'/api/videos/channels/download/{simple_channel.id}')
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, assert_video_search):

View file

@ -81,7 +81,8 @@ def test_is_valid_poster(ext, expected, test_directory):
assert is_valid_poster(image_path) == expected, f'is_valid_poster({image_path}) should be {expected}'
def test_update_view_count(test_session, channel_factory, video_factory):
@pytest.mark.asyncio
async def test_update_view_count(test_session, channel_factory, video_factory):
def check_view_counts(view_counts):
for source_id, view_count in view_counts.items():
video = test_session.query(Video).filter_by(source_id=source_id).one()
@ -101,22 +102,22 @@ def test_update_view_count(test_session, channel_factory, video_factory):
check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None})
# Channel 1 is updated, the other channels are left alone.
update_view_counts(channel1.id)
await update_view_counts(channel1.id)
check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None})
# Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored.
update_view_counts(channel2.id)
await update_view_counts(channel2.id)
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None})
# All videos are updated.
update_view_counts(channel3.id)
await update_view_counts(channel3.id)
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
# An outdated view count will be overwritten.
vid = test_session.query(Video).filter_by(id=1).one()
vid.view_count = 8
check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14})
update_view_counts(channel1.id)
await update_view_counts(channel1.id)
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})

View file

@ -148,7 +148,7 @@ def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixtu
_, resp = api_app.test_client.post('/api/files/refresh')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
_, resp = api_app.test_client.post(f'/api/videos/download/{simple_channel.id}')
_, resp = api_app.test_client.post(f'/api/videos/channels/download/{simple_channel.id}')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']