mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Adding toggle to Channel page to stop downloading comments for certain channels.
This commit is contained in:
parent
47d4d72bf8
commit
581d5eaf6e
20 changed files with 150 additions and 77 deletions
|
|
@ -5,5 +5,5 @@ from wrolpi.errors import APIError
|
|||
|
||||
class InvalidArchive(APIError):
|
||||
code = 'INVALID_ARCHIVE'
|
||||
message = 'The archive is invalid. See server logs.'
|
||||
summary = 'The archive is invalid. See server logs.'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ from wrolpi.errors import APIError
|
|||
|
||||
class NoInventories(APIError):
|
||||
code = 'NO_INVENTORIES'
|
||||
message = 'No Inventories'
|
||||
summary = 'No Inventories'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class InventoriesVersionMismatch(APIError):
|
||||
code = 'INVENTORIES_VERSION_MISMATCH'
|
||||
message = 'Inventories version in the DB does not match the inventories config'
|
||||
summary = 'Inventories version in the DB does not match the inventories config'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
|
|
|||
|
|
@ -5,17 +5,17 @@ from wrolpi.errors import APIError
|
|||
|
||||
class InvalidOTP(APIError):
|
||||
code = 'INVALID_OTP'
|
||||
message = 'OTP has invalid characters'
|
||||
summary = 'OTP has invalid characters'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class InvalidPlaintext(APIError):
|
||||
code = 'INVALID_PLAINTEXT'
|
||||
message = 'Plaintext has invalid characters'
|
||||
summary = 'Plaintext has invalid characters'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class InvalidCiphertext(APIError):
|
||||
code = 'INVALID_CIPHERTEXT'
|
||||
message = 'Ciphertext has invalid characters'
|
||||
summary = 'Ciphertext has invalid characters'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
|
|
|||
|
|
@ -127,10 +127,10 @@ async def test_channel_conflicts(async_client, test_session, test_directory):
|
|||
request, response = await _post_channel(new_channel)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json == {'cause': {'code': 'CHANNEL_NAME_CONFLICT',
|
||||
'error': 'The channel name is already taken.',
|
||||
'error': 'Bad Request',
|
||||
'message': 'The channel name is already taken.'},
|
||||
'code': 'VALIDATION_ERROR',
|
||||
'error': 'Could not validate the contents of the request',
|
||||
'error': 'Bad Request',
|
||||
'message': 'Could not validate the contents of the request'}
|
||||
|
||||
# Directory was already used
|
||||
|
|
@ -141,10 +141,10 @@ async def test_channel_conflicts(async_client, test_session, test_directory):
|
|||
request, response = await _post_channel(new_channel)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json == {'cause': {'code': 'CHANNEL_DIRECTORY_CONFLICT',
|
||||
'error': 'The directory is already used by another channel.',
|
||||
'error': 'Bad Request',
|
||||
'message': 'The directory is already used by another channel.'},
|
||||
'code': 'VALIDATION_ERROR',
|
||||
'error': 'Could not validate the contents of the request',
|
||||
'error': 'Bad Request',
|
||||
'message': 'Could not validate the contents of the request'}
|
||||
|
||||
# URL is already used
|
||||
|
|
@ -158,10 +158,10 @@ async def test_channel_conflicts(async_client, test_session, test_directory):
|
|||
request, response = await _post_channel(new_channel)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json == {'cause': {'code': 'CHANNEL_URL_CONFLICT',
|
||||
'error': 'The URL is already used by another channel.',
|
||||
'error': 'Bad Request',
|
||||
'message': 'The URL is already used by another channel.'},
|
||||
'code': 'VALIDATION_ERROR',
|
||||
'error': 'Could not validate the contents of the request',
|
||||
'error': 'Bad Request',
|
||||
'message': 'Could not validate the contents of the request'}
|
||||
|
||||
|
||||
|
|
@ -170,21 +170,15 @@ def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_direc
|
|||
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
||||
pathlib.Path(channel_directory).mkdir()
|
||||
|
||||
new_channel = {
|
||||
'name': 'Fooz',
|
||||
'directory': channel_directory,
|
||||
}
|
||||
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
new_channel = dict(name='Fooz', directory=channel_directory)
|
||||
request, response = test_client.post('/api/videos/channels', json=new_channel)
|
||||
assert response.status_code == HTTPStatus.CREATED, response.json
|
||||
location = response.headers['Location']
|
||||
|
||||
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
||||
pathlib.Path(channel_directory2).mkdir()
|
||||
new_channel = {
|
||||
'name': 'Barz',
|
||||
'directory': channel_directory2,
|
||||
}
|
||||
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
new_channel = dict(name='Barz', directory=channel_directory2)
|
||||
request, response = test_client.post('/api/videos/channels', json=new_channel)
|
||||
assert response.status_code == HTTPStatus.CREATED, response.json
|
||||
assert location != response.headers['Location']
|
||||
|
||||
|
|
|
|||
|
|
@ -5,47 +5,47 @@ from wrolpi.errors import APIError
|
|||
|
||||
class UnknownVideo(APIError):
|
||||
code = 'UNKNOWN_VIDEO'
|
||||
message = 'The video could not be found.'
|
||||
summary = 'The video could not be found.'
|
||||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class UnknownChannel(APIError):
|
||||
code = 'UNKNOWN_CHANNEL'
|
||||
message = 'The channel could not be found.'
|
||||
summary = 'The channel could not be found.'
|
||||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class InvalidChannel(APIError):
|
||||
code = 'INVALID_CHANNEL'
|
||||
message = 'The channel is not supported'
|
||||
summary = 'The channel is not supported'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class ChannelNameConflict(APIError):
|
||||
code = 'CHANNEL_NAME_CONFLICT'
|
||||
message = 'The channel name is already taken.'
|
||||
summary = 'The channel name is already taken.'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class ChannelURLConflict(APIError):
|
||||
code = 'CHANNEL_URL_CONFLICT'
|
||||
message = 'The URL is already used by another channel.'
|
||||
summary = 'The URL is already used by another channel.'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class ChannelDirectoryConflict(APIError):
|
||||
code = 'CHANNEL_DIRECTORY_CONFLICT'
|
||||
message = 'The directory is already used by another channel.'
|
||||
summary = 'The directory is already used by another channel.'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class ChannelSourceIdConflict(APIError):
|
||||
code = 'CHANNEL_SOURCE_ID_CONFLICT'
|
||||
message = 'Search is empty, search_str must have content.'
|
||||
summary = 'Search is empty, search_str must have content.'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class ChannelURLEmpty(APIError):
|
||||
code = 'CHANNEL_URL_EMPTY'
|
||||
message = 'This channel has no URL'
|
||||
summary = 'This channel has no URL'
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
|
|
|||
|
|
@ -297,7 +297,7 @@ class ChannelsConfig(ConfigFile):
|
|||
|
||||
if not channel.source_id and channel.url and flags.have_internet.is_set():
|
||||
# If we can download from a channel, we must have its source_id.
|
||||
if download_manager.can_download:
|
||||
if download_manager.can_download and channel.download_missing_data:
|
||||
logger.info(f'Fetching channel source id for {channel}')
|
||||
background_task(fetch_channel_source_id(channel.id))
|
||||
|
||||
|
|
|
|||
|
|
@ -528,6 +528,7 @@ class Channel(ModelHelper, Base):
|
|||
directory: pathlib.Path = Column(MediaPathType)
|
||||
generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow.
|
||||
calculate_duration = Column(Boolean, default=True) # use ffmpeg to extract duration (slower than info json).
|
||||
download_missing_data = Column(Boolean, default=True) # fetch missing data like `source_id` and video comments.
|
||||
source_id = Column(String) # the ID from the source website.
|
||||
refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed.
|
||||
|
||||
|
|
@ -633,6 +634,7 @@ class Channel(ModelHelper, Base):
|
|||
config = dict(
|
||||
calculate_duration=self.calculate_duration,
|
||||
directory=str(self.directory),
|
||||
download_missing_data=self.download_missing_data,
|
||||
downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads],
|
||||
generate_posters=self.generate_posters,
|
||||
name=self.name,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ class ChannelPostRequest:
|
|||
name: str
|
||||
directory: str
|
||||
calculate_duration: Optional[bool] = None
|
||||
download_missing_data: Optional[bool] = True
|
||||
generate_posters: Optional[bool] = None
|
||||
source_id: Optional[str] = None
|
||||
tag_name: Optional[str] = None
|
||||
|
|
@ -26,6 +27,7 @@ class ChannelPostRequest:
|
|||
class ChannelPutRequest:
|
||||
calculate_duration: Optional[bool] = None
|
||||
directory: Optional[str] = None
|
||||
download_missing_data: Optional[bool] = True
|
||||
generate_posters: Optional[bool] = None
|
||||
mkdir: Optional[bool] = None
|
||||
name: Optional[str] = None
|
||||
|
|
|
|||
|
|
@ -288,6 +288,25 @@ async def test_import_channel_delete_missing_channels(await_switches, test_sessi
|
|||
assert str(channel2.directory) not in test_channels_config.read_text()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_channel_download_comments(await_switches, test_session, channel_factory,
|
||||
test_channels_config):
|
||||
channel = channel_factory()
|
||||
assert channel.download_missing_data is True
|
||||
|
||||
# download_missing_data is saved to the config.
|
||||
save_channels_config()
|
||||
await await_switches()
|
||||
assert 'download_missing_data: true' in test_channels_config.read_text()
|
||||
|
||||
# change download_missing_data to False, channel should be updated on import.
|
||||
contents = test_channels_config.read_text()
|
||||
contents = contents.replace('download_missing_data: true', 'download_missing_data: false')
|
||||
test_channels_config.write_text(contents)
|
||||
import_channels_config()
|
||||
assert channel.download_missing_data is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ffprobe_json(async_client, video_file, corrupted_video_file):
|
||||
content = await common.ffprobe_json(video_file)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import yt_dlp
|
|||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.videos.models import Video
|
||||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.common import logger, limit_concurrent, wrol_mode_check
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, optional_session
|
||||
|
|
@ -209,7 +209,13 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
|
|||
one_month_ago > FileGroup.published_datetime,
|
||||
FileGroup.published_datetime == None,
|
||||
),
|
||||
# Do not download a Videos with a Channel and Channel.download_missing_data is False.
|
||||
or_(
|
||||
Channel.download_missing_data == True,
|
||||
Channel.id == None,
|
||||
)
|
||||
).join(FileGroup) \
|
||||
.outerjoin(Channel) \
|
||||
.order_by(FileGroup.published_datetime.nullsfirst()) \
|
||||
.limit(limit)
|
||||
video_urls = [i.file_group.url for i in videos]
|
||||
|
|
|
|||
|
|
@ -5,23 +5,23 @@ from wrolpi.errors import APIError
|
|||
|
||||
class UnknownZim(APIError):
|
||||
code = 'UNKNOWN_ZIM'
|
||||
message = 'Failed to find the Zim'
|
||||
summary = 'Failed to find the Zim'
|
||||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class UnknownZimEntry(APIError):
|
||||
code = 'UNKNOWN_ZIM_ENTRY'
|
||||
message = 'Failed to find the Zim entry at that path'
|
||||
summary = 'Failed to find the Zim entry at that path'
|
||||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class UnknownZimTagEntry(APIError):
|
||||
code = 'UNKNOWN_ZIM_TAG_ENTRY'
|
||||
message = 'Failed to find the TagZimEntry'
|
||||
summary = 'Failed to find the TagZimEntry'
|
||||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class UnknownZimSubscription(APIError):
|
||||
code = 'UNKNOWN_ZIM_SUBSCRIPTION'
|
||||
message = 'Failed to find the ZimSubscription'
|
||||
summary = 'Failed to find the ZimSubscription'
|
||||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue