From a96d6f97f3cd7d28703094671fc2ddeea93a6d98 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Fri, 7 Jan 2022 14:41:28 -0700
Subject: [PATCH] A Channel must be refreshed before download.

---
 .../versions/f8c65e7e6802_channel_refresh.py  | 31 +++++++++++++++++++
 modules/videos/channel/test/test_api.py       | 18 +++++++++++
 modules/videos/downloader.py                  | 21 ++-----------
 modules/videos/lib.py                         |  5 +++
 modules/videos/models.py                      |  1 +
 5 files changed, 58 insertions(+), 18 deletions(-)
 create mode 100644 alembic/versions/f8c65e7e6802_channel_refresh.py

diff --git a/alembic/versions/f8c65e7e6802_channel_refresh.py b/alembic/versions/f8c65e7e6802_channel_refresh.py
new file mode 100644
index 00000000..8a34aedd
--- /dev/null
+++ b/alembic/versions/f8c65e7e6802_channel_refresh.py
@@ -0,0 +1,31 @@
+"""Channel.refresh
+
+Revision ID: f8c65e7e6802
+Revises: 62acddc55091
+Create Date: 2022-01-07 14:28:21.787842
+
+"""
+import os
+from alembic import op
+from sqlalchemy.orm import Session
+
+
+# revision identifiers, used by Alembic.
+revision = 'f8c65e7e6802'
+down_revision = '62acddc55091'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    session.execute('ALTER TABLE channel ADD COLUMN refreshed BOOL DEFAULT false')
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS refreshed')
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index 2e985ac1..f333d636 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -607,3 +607,21 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory):
     request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
     assert response.status_code == HTTPStatus.CREATED, response.json
     assert location != response.headers['Location']
+
+
+def test_download_channel_no_refresh(test_session, download_channel, video_download_manager):
+    """
+    A Channel cannot be downloaded until it has been refreshed.
+    """
+    def check_refreshed(expected: bool):
+        channel = test_session.query(Channel).one()
+        assert channel.refreshed == expected
+
+    check_refreshed(False)
+    test_session.commit()
+
+    with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
+        mock_extract_info.return_value = {'entries': [], 'url': 'foo'}
+        video_download_manager.do_downloads_sync()
+
+    check_refreshed(True)
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 65c4f6c3..e56da197 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -20,7 +20,7 @@ from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownload
 from wrolpi.vars import PYTEST
 from .channel.lib import create_channel, get_channel
 from .common import load_downloader_config, apply_info_json, get_channel_source_id
-from .lib import upsert_video, _refresh_videos
+from .lib import upsert_video, _refresh_videos, refresh_channel_videos
 from .models import Video, Channel
 from .video_url_resolver import video_url_resolver
 
@@ -400,24 +400,9 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
 
     # Check that each channel has some videos.  We can't be sure what is missing if we don't know what we have.
     for channel in channels:
-        refresh = False
-        with get_db_curs() as curs:
-            curs.execute('''
-                SELECT
-                    COUNT(v.id)
-                FROM
-                    channel c
-                    LEFT OUTER JOIN video v on c.id = v.channel_id
-                WHERE
-                    c.id = %s
-            ''', (channel.id,))
-            video_count = curs.fetchall()[0]
-            if video_count == 0:
-                # No videos for this channel.  We can't be sure if any videos are missing without a refresh.
-                refresh = True
-        if refresh:
+        if channel.refreshed is False:
             # Refresh this channel's videos.
-            _refresh_videos([channel.link])
+            refresh_channel_videos(channel)
 
     channels = {i.id: i for i in channels}
 
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 690a54d9..53c65310 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -176,6 +176,11 @@ def refresh_channel_videos(channel: Channel):
 
     logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ')
 
+    with get_db_session(commit=True) as session:
+        # All videos have been discovered, mark the channel as refreshed.
+        channel.refreshed = True
+        session.add(channel)
+
     apply_info_json(channel.id)
 
 
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 43fa213c..230da750 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -261,6 +261,7 @@ class Channel(ModelHelper, Base):
     calculate_duration = Column(Boolean)
     download_frequency = Column(Integer)
     source_id = Column(String)
+    refreshed = Column(Boolean, default=False)
 
     info_json = Column(JSON)
     info_date = Column(Date)