From df5eee27d68b7b1743c38bf8a47f6dea8b31d72c Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 11 Oct 2019 16:30:59 -0600 Subject: [PATCH] Working refresh streaming in. --- wrolpi/plugins/videos/api.py | 34 +++++++++++++++++-- .../videos/templates/channels_settings.html | 2 +- wrolpi/plugins/videos/test/test_api.py | 31 +++++++++++++++-- wrolpi/plugins/videos/v1.sql | 8 +++++ 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/wrolpi/plugins/videos/api.py b/wrolpi/plugins/videos/api.py index 73144006..8e3ce77a 100644 --- a/wrolpi/plugins/videos/api.py +++ b/wrolpi/plugins/videos/api.py @@ -1,10 +1,12 @@ import json import pathlib +import time import cherrypy +import psycopg2 from dictorm import DictDB -from wrolpi.common import sanitize_link +from wrolpi.common import sanitize_link, get_db_context from wrolpi.plugins.videos.common import get_conflicting_channels from wrolpi.plugins.videos.downloader import insert_video from wrolpi.plugins.videos.main import logger @@ -38,10 +40,24 @@ class SettingsAPI(object): class Refresh(object): @cherrypy.tools.db() - def GET(self, db: DictDB): + def POST(self, db: DictDB): refresh_videos(db) return json.dumps({'success': 'Videos refreshed'}) + def GET(self): + cherrypy.response.headers['Content-Type'] = 'text/event-stream' + + def streamer(): + with get_db_context() as (db_conn, db): + Video = db['video'] + for video in Video.get_where(): + yield f'{video["name"]}\n' + time.sleep(0.5) + + return streamer() + + GET._cp_config = {'response.stream': True} + @cherrypy.expose class ChannelAPI(object): @@ -177,19 +193,31 @@ def refresh_channel_videos(db, channel): logger.info(f'{channel["name"]}: Added {len(new_videos)} new videos, {len(existing_paths)} already existed.') +def insert_status(curs, status): + curs.execute('INSERT INTO refresh_status (status) VALUES (%s);', (status,)) + + def refresh_videos(db: DictDB): logger.info('Refreshing video list') Channel = db['channel'] # Remove any videos that don't exist curs = db.get_cursor() + insert_status(curs, 'Refreshing video list') + deleted_any = False curs.execute('SELECT id, video_path FROM video') existing_videos = curs.fetchall() for video in existing_videos: if not pathlib.Path(video['video_path']).is_file(): + deleted_any = True curs.execute('DELETE FROM video WHERE id = %s', (video['id'],)) - return + + if deleted_any: + insert_status(curs, 'Removed records of non-existent videos') + else: + insert_status(curs, 'All video records link to real files') for channel in Channel.get_where(): + insert_status(curs, f'Searching for videos for channel {channel["name"]}') with db.transaction(commit=True): refresh_channel_videos(db, channel) diff --git a/wrolpi/plugins/videos/templates/channels_settings.html b/wrolpi/plugins/videos/templates/channels_settings.html index 00c3084a..87d34d99 100644 --- a/wrolpi/plugins/videos/templates/channels_settings.html +++ b/wrolpi/plugins/videos/templates/channels_settings.html @@ -37,7 +37,7 @@
- {{ get_form_button('refresh', 'dark', 'Refresh Video Files') }} + {{ post_form_button('refresh', 'dark', 'Refresh Video Files') }}
diff --git a/wrolpi/plugins/videos/test/test_api.py b/wrolpi/plugins/videos/test/test_api.py index 8a190909..9809a08f 100644 --- a/wrolpi/plugins/videos/test/test_api.py +++ b/wrolpi/plugins/videos/test/test_api.py @@ -3,7 +3,9 @@ import tempfile import unittest from shutil import copyfile +import cherrypy import mock +from cherrypy.test import helper from wrolpi.common import get_db_context from wrolpi.plugins.videos.api import APIRoot @@ -46,16 +48,18 @@ class TestAPI(unittest.TestCase): def test_refresh(self): api = APIRoot() with get_db_context() as (db_conn, db): - Video, Channel = db['video'], db['channel'] + Video, Channel, RefreshStatus = db['video'], db['channel'], db['refresh_status'] import_settings_config() # Insert a bogus video, it should be removed bogus = Video(video_path='bar').flush() assert bogus['id'], 'Failed to insert a bogus video for removal' - api.settings.refresh.GET(db) + api.settings.refresh.POST(db) self.assertEqual(Channel.count(), 1) self.assertEqual(Video.count(), 1) + # Some statuses were saved + self.assertGreater(RefreshStatus.count(), 0) @test_db_wrapper def test_channel(self): @@ -102,3 +106,26 @@ class TestAPI(unittest.TestCase): # Delete the new channel result = api.channel.DELETE('examplechannel1', db) self.assertIn('success', result) + + @test_db_wrapper + def test_refresh(self): + # Insert some statuses + with get_db_context() as (db_conn, db): + RefreshStatus = db['refresh_status'] + example_statuses = [ + 'Foo', + 'Bar', + 'Baz', + 'Refresh complete', + 'this should not be shown' + ] + for status in example_statuses: + RefreshStatus(status=status).flush() + + response = self.getPage('/api/videos/settings/refresh') + statuses = '\n'.join([ + 'Foo', + 'Bar', + 'Baz', + 'Refresh complete' + ]) diff --git a/wrolpi/plugins/videos/v1.sql b/wrolpi/plugins/videos/v1.sql index ae95e832..7e0dae84 100644 --- a/wrolpi/plugins/videos/v1.sql +++ b/wrolpi/plugins/videos/v1.sql @@ -34,3 +34,11 @@ CREATE TABLE video ALTER TABLE video ADD COLUMN channel_id INTEGER REFERENCES channel (id); + +DROP TABLE IF EXISTS refresh_status; +CREATE TABLE refresh_status +( + id SERIAL PRIMARY KEY, + status_dt TIMESTAMP DEFAULT current_timestamp, + status TEXT +);