Adding toggle to Channel page to stop downloading comments for certain channels.

This commit is contained in:
Roland 2025-02-19 14:52:50 -07:00
parent 47d4d72bf8
commit 581d5eaf6e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
20 changed files with 150 additions and 77 deletions

View file

@ -5,5 +5,5 @@ from wrolpi.errors import APIError
class InvalidArchive(APIError):
code = 'INVALID_ARCHIVE'
message = 'The archive is invalid. See server logs.'
summary = 'The archive is invalid. See server logs.'
status_code = HTTPStatus.BAD_REQUEST

View file

@ -5,11 +5,11 @@ from wrolpi.errors import APIError
class NoInventories(APIError):
code = 'NO_INVENTORIES'
message = 'No Inventories'
summary = 'No Inventories'
status_code = HTTPStatus.BAD_REQUEST
class InventoriesVersionMismatch(APIError):
code = 'INVENTORIES_VERSION_MISMATCH'
message = 'Inventories version in the DB does not match the inventories config'
summary = 'Inventories version in the DB does not match the inventories config'
status_code = HTTPStatus.BAD_REQUEST

View file

@ -5,17 +5,17 @@ from wrolpi.errors import APIError
class InvalidOTP(APIError):
code = 'INVALID_OTP'
message = 'OTP has invalid characters'
summary = 'OTP has invalid characters'
status_code = HTTPStatus.BAD_REQUEST
class InvalidPlaintext(APIError):
code = 'INVALID_PLAINTEXT'
message = 'Plaintext has invalid characters'
summary = 'Plaintext has invalid characters'
status_code = HTTPStatus.BAD_REQUEST
class InvalidCiphertext(APIError):
code = 'INVALID_CIPHERTEXT'
message = 'Ciphertext has invalid characters'
summary = 'Ciphertext has invalid characters'
status_code = HTTPStatus.BAD_REQUEST

View file

@ -127,10 +127,10 @@ async def test_channel_conflicts(async_client, test_session, test_directory):
request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_NAME_CONFLICT',
'error': 'The channel name is already taken.',
'error': 'Bad Request',
'message': 'The channel name is already taken.'},
'code': 'VALIDATION_ERROR',
'error': 'Could not validate the contents of the request',
'error': 'Bad Request',
'message': 'Could not validate the contents of the request'}
# Directory was already used
@ -141,10 +141,10 @@ async def test_channel_conflicts(async_client, test_session, test_directory):
request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_DIRECTORY_CONFLICT',
'error': 'The directory is already used by another channel.',
'error': 'Bad Request',
'message': 'The directory is already used by another channel.'},
'code': 'VALIDATION_ERROR',
'error': 'Could not validate the contents of the request',
'error': 'Bad Request',
'message': 'Could not validate the contents of the request'}
# URL is already used
@ -158,10 +158,10 @@ async def test_channel_conflicts(async_client, test_session, test_directory):
request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_URL_CONFLICT',
'error': 'The URL is already used by another channel.',
'error': 'Bad Request',
'message': 'The URL is already used by another channel.'},
'code': 'VALIDATION_ERROR',
'error': 'Could not validate the contents of the request',
'error': 'Bad Request',
'message': 'Could not validate the contents of the request'}
@ -170,21 +170,15 @@ def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_direc
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = {
'name': 'Fooz',
'directory': channel_directory,
}
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
new_channel = dict(name='Fooz', directory=channel_directory)
request, response = test_client.post('/api/videos/channels', json=new_channel)
assert response.status_code == HTTPStatus.CREATED, response.json
location = response.headers['Location']
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory2).mkdir()
new_channel = {
'name': 'Barz',
'directory': channel_directory2,
}
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
new_channel = dict(name='Barz', directory=channel_directory2)
request, response = test_client.post('/api/videos/channels', json=new_channel)
assert response.status_code == HTTPStatus.CREATED, response.json
assert location != response.headers['Location']

View file

@ -5,47 +5,47 @@ from wrolpi.errors import APIError
class UnknownVideo(APIError):
code = 'UNKNOWN_VIDEO'
message = 'The video could not be found.'
summary = 'The video could not be found.'
status_code = HTTPStatus.NOT_FOUND
class UnknownChannel(APIError):
code = 'UNKNOWN_CHANNEL'
message = 'The channel could not be found.'
summary = 'The channel could not be found.'
status_code = HTTPStatus.NOT_FOUND
class InvalidChannel(APIError):
code = 'INVALID_CHANNEL'
message = 'The channel is not supported'
summary = 'The channel is not supported'
status_code = HTTPStatus.BAD_REQUEST
class ChannelNameConflict(APIError):
code = 'CHANNEL_NAME_CONFLICT'
message = 'The channel name is already taken.'
summary = 'The channel name is already taken.'
status_code = HTTPStatus.BAD_REQUEST
class ChannelURLConflict(APIError):
code = 'CHANNEL_URL_CONFLICT'
message = 'The URL is already used by another channel.'
summary = 'The URL is already used by another channel.'
status_code = HTTPStatus.BAD_REQUEST
class ChannelDirectoryConflict(APIError):
code = 'CHANNEL_DIRECTORY_CONFLICT'
message = 'The directory is already used by another channel.'
summary = 'The directory is already used by another channel.'
status_code = HTTPStatus.BAD_REQUEST
class ChannelSourceIdConflict(APIError):
code = 'CHANNEL_SOURCE_ID_CONFLICT'
message = 'Search is empty, search_str must have content.'
summary = 'Search is empty, search_str must have content.'
status_code = HTTPStatus.BAD_REQUEST
class ChannelURLEmpty(APIError):
code = 'CHANNEL_URL_EMPTY'
message = 'This channel has no URL'
summary = 'This channel has no URL'
status_code = HTTPStatus.BAD_REQUEST

View file

@ -297,7 +297,7 @@ class ChannelsConfig(ConfigFile):
if not channel.source_id and channel.url and flags.have_internet.is_set():
# If we can download from a channel, we must have its source_id.
if download_manager.can_download:
if download_manager.can_download and channel.download_missing_data:
logger.info(f'Fetching channel source id for {channel}')
background_task(fetch_channel_source_id(channel.id))

View file

@ -528,6 +528,7 @@ class Channel(ModelHelper, Base):
directory: pathlib.Path = Column(MediaPathType)
generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow.
calculate_duration = Column(Boolean, default=True) # use ffmpeg to extract duration (slower than info json).
download_missing_data = Column(Boolean, default=True) # fetch missing data like `source_id` and video comments.
source_id = Column(String) # the ID from the source website.
refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed.
@ -633,6 +634,7 @@ class Channel(ModelHelper, Base):
config = dict(
calculate_duration=self.calculate_duration,
directory=str(self.directory),
download_missing_data=self.download_missing_data,
downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads],
generate_posters=self.generate_posters,
name=self.name,

View file

@ -10,6 +10,7 @@ class ChannelPostRequest:
name: str
directory: str
calculate_duration: Optional[bool] = None
download_missing_data: Optional[bool] = True
generate_posters: Optional[bool] = None
source_id: Optional[str] = None
tag_name: Optional[str] = None
@ -26,6 +27,7 @@ class ChannelPostRequest:
class ChannelPutRequest:
calculate_duration: Optional[bool] = None
directory: Optional[str] = None
download_missing_data: Optional[bool] = True
generate_posters: Optional[bool] = None
mkdir: Optional[bool] = None
name: Optional[str] = None

View file

@ -288,6 +288,25 @@ async def test_import_channel_delete_missing_channels(await_switches, test_sessi
assert str(channel2.directory) not in test_channels_config.read_text()
@pytest.mark.asyncio
async def test_import_channel_download_comments(await_switches, test_session, channel_factory,
test_channels_config):
channel = channel_factory()
assert channel.download_missing_data is True
# download_missing_data is saved to the config.
save_channels_config()
await await_switches()
assert 'download_missing_data: true' in test_channels_config.read_text()
# change download_missing_data to False, channel should be updated on import.
contents = test_channels_config.read_text()
contents = contents.replace('download_missing_data: true', 'download_missing_data: false')
test_channels_config.write_text(contents)
import_channels_config()
assert channel.download_missing_data is False
@pytest.mark.asyncio
async def test_ffprobe_json(async_client, video_file, corrupted_video_file):
content = await common.ffprobe_json(video_file)

View file

@ -7,7 +7,7 @@ import yt_dlp
from sqlalchemy import or_
from sqlalchemy.orm import Session
from modules.videos.models import Video
from modules.videos.models import Video, Channel
from wrolpi.common import logger, limit_concurrent, wrol_mode_check
from wrolpi.dates import now
from wrolpi.db import get_db_session, optional_session
@ -209,7 +209,13 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
one_month_ago > FileGroup.published_datetime,
FileGroup.published_datetime == None,
),
# Do not download a Videos with a Channel and Channel.download_missing_data is False.
or_(
Channel.download_missing_data == True,
Channel.id == None,
)
).join(FileGroup) \
.outerjoin(Channel) \
.order_by(FileGroup.published_datetime.nullsfirst()) \
.limit(limit)
video_urls = [i.file_group.url for i in videos]

View file

@ -5,23 +5,23 @@ from wrolpi.errors import APIError
class UnknownZim(APIError):
code = 'UNKNOWN_ZIM'
message = 'Failed to find the Zim'
summary = 'Failed to find the Zim'
status_code = HTTPStatus.NOT_FOUND
class UnknownZimEntry(APIError):
code = 'UNKNOWN_ZIM_ENTRY'
message = 'Failed to find the Zim entry at that path'
summary = 'Failed to find the Zim entry at that path'
status_code = HTTPStatus.NOT_FOUND
class UnknownZimTagEntry(APIError):
code = 'UNKNOWN_ZIM_TAG_ENTRY'
message = 'Failed to find the TagZimEntry'
summary = 'Failed to find the TagZimEntry'
status_code = HTTPStatus.NOT_FOUND
class UnknownZimSubscription(APIError):
code = 'UNKNOWN_ZIM_SUBSCRIPTION'
message = 'Failed to find the ZimSubscription'
summary = 'Failed to find the ZimSubscription'
status_code = HTTPStatus.NOT_FOUND