Working refresh streaming in.

wrolpi/plugins/videos/api.py

@@ -1,10 +1,12 @@
 import json
 import pathlib
+import time
 
 import cherrypy
+import psycopg2
 from dictorm import DictDB
 
-from wrolpi.common import sanitize_link
+from wrolpi.common import sanitize_link, get_db_context
 from wrolpi.plugins.videos.common import get_conflicting_channels
 from wrolpi.plugins.videos.downloader import insert_video
 from wrolpi.plugins.videos.main import logger
@@ -38,10 +40,24 @@ class SettingsAPI(object):
 class Refresh(object):
 
     @cherrypy.tools.db()
-    def GET(self, db: DictDB):
+    def POST(self, db: DictDB):
         refresh_videos(db)
         return json.dumps({'success': 'Videos refreshed'})
 
+    def GET(self):
+        cherrypy.response.headers['Content-Type'] = 'text/event-stream'
+
+        def streamer():
+            with get_db_context() as (db_conn, db):
+                Video = db['video']
+                for video in Video.get_where():
+                    yield f'{video["name"]}\n'
+                    time.sleep(0.5)
+
+        return streamer()
+
+    GET._cp_config = {'response.stream': True}
+
 
 @cherrypy.expose
 class ChannelAPI(object):
@@ -177,19 +193,31 @@ def refresh_channel_videos(db, channel):
     logger.info(f'{channel["name"]}: Added {len(new_videos)} new videos, {len(existing_paths)} already existed.')
 
 
+def insert_status(curs, status):
+    curs.execute('INSERT INTO refresh_status (status) VALUES (%s);', (status,))
+
+
 def refresh_videos(db: DictDB):
     logger.info('Refreshing video list')
     Channel = db['channel']
 
     # Remove any videos that don't exist
     curs = db.get_cursor()
+    insert_status(curs, 'Refreshing video list')
+    deleted_any = False
     curs.execute('SELECT id, video_path FROM video')
     existing_videos = curs.fetchall()
     for video in existing_videos:
         if not pathlib.Path(video['video_path']).is_file():
+            deleted_any = True
             curs.execute('DELETE FROM video WHERE id = %s', (video['id'],))
-        return
+
+    if deleted_any:
+        insert_status(curs, 'Removed records of non-existent videos')
+    else:
+        insert_status(curs, 'All video records link to real files')
 
     for channel in Channel.get_where():
+        insert_status(curs, f'Searching for videos for channel {channel["name"]}')
         with db.transaction(commit=True):
             refresh_channel_videos(db, channel)

wrolpi/plugins/videos/templates/channels_settings.html

@@ -37,7 +37,7 @@
                 <div class="row">
                     <div class="col-4">
                         <form id="refresh" action="/api/videos/settings/refresh">
-                            {{ get_form_button('refresh', 'dark', 'Refresh Video Files') }}
+                            {{ post_form_button('refresh', 'dark', 'Refresh Video Files') }}
                         </form>
                     </div>
                     <div class="col-8">

wrolpi/plugins/videos/test/test_api.py

@@ -3,7 +3,9 @@ import tempfile
 import unittest
 from shutil import copyfile
 
+import cherrypy
 import mock
+from cherrypy.test import helper
 
 from wrolpi.common import get_db_context
 from wrolpi.plugins.videos.api import APIRoot
@@ -46,16 +48,18 @@ class TestAPI(unittest.TestCase):
     def test_refresh(self):
         api = APIRoot()
         with get_db_context() as (db_conn, db):
-            Video, Channel = db['video'], db['channel']
+            Video, Channel, RefreshStatus = db['video'], db['channel'], db['refresh_status']
             import_settings_config()
 
             # Insert a bogus video, it should be removed
             bogus = Video(video_path='bar').flush()
             assert bogus['id'], 'Failed to insert a bogus video for removal'
 
-            api.settings.refresh.GET(db)
+            api.settings.refresh.POST(db)
             self.assertEqual(Channel.count(), 1)
             self.assertEqual(Video.count(), 1)
+            # Some statuses were saved
+            self.assertGreater(RefreshStatus.count(), 0)
 
     @test_db_wrapper
     def test_channel(self):
@@ -102,3 +106,26 @@ class TestAPI(unittest.TestCase):
             # Delete the new channel
             result = api.channel.DELETE('examplechannel1', db)
             self.assertIn('success', result)
+
+    @test_db_wrapper
+    def test_refresh(self):
+        # Insert some statuses
+        with get_db_context() as (db_conn, db):
+            RefreshStatus = db['refresh_status']
+            example_statuses = [
+                'Foo',
+                'Bar',
+                'Baz',
+                'Refresh complete',
+                'this should not be shown'
+            ]
+            for status in example_statuses:
+                RefreshStatus(status=status).flush()
+
+        response = self.getPage('/api/videos/settings/refresh')
+        statuses = '\n'.join([
+            'Foo',
+            'Bar',
+            'Baz',
+            'Refresh complete'
+        ])

wrolpi/plugins/videos/v1.sql

@@ -34,3 +34,11 @@ CREATE TABLE video
 
 ALTER TABLE video
     ADD COLUMN channel_id INTEGER REFERENCES channel (id);
+
+DROP TABLE IF EXISTS refresh_status;
+CREATE TABLE refresh_status
+(
+    id        SERIAL PRIMARY KEY,
+    status_dt TIMESTAMP DEFAULT current_timestamp,
+    status    TEXT
+);