diff --git a/modules/repos/lib.py b/modules/repos/lib.py index 49e3e528..b56ef585 100644 --- a/modules/repos/lib.py +++ b/modules/repos/lib.py @@ -1562,7 +1562,9 @@ class ReposConfig(ConfigFile): repos_config = ReposConfig() -register_ownership(ReposConfig.file_name, Repository) +# Only the columns repos.yaml saves (not a fetch's commit, size, ...); its Collection saves the rest. +register_ownership(ReposConfig.file_name, Repository, + columns={'url', 'mode', 'branch', 'submodules', 'clone_token', 'collection_id'}) def get_repos_config() -> ReposConfig: diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 292224e8..2a64b7aa 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -530,7 +530,10 @@ class ChannelsConfig(ConfigFile): CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig() -register_ownership(ChannelsConfig.file_name, Channel) +# Only the columns channels.yaml saves (not a refresh's counts, info_json, ...); its Collection saves the rest. +register_ownership(ChannelsConfig.file_name, Channel, + columns={'url', 'source_id', 'generate_posters', 'calculate_duration', 'download_missing_data', + 'collection_id'}) TEST_CHANNELS_CONFIG: ChannelsConfig = None diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index df04a034..047cb35c 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -1316,9 +1316,11 @@ class CollectionItem(ModelHelper, Base): return data -# Each kind of Collection is saved by its own config. +# Each kind of Collection is saved by its own config, which saves these columns (not item_count, total_size, ...). +COLLECTION_CONFIG_COLUMNS = {'name', 'kind', 'description', 'directory', 'tag_id', 'file_format'} for _kind, _config in (('channel', 'channels.yaml'), ('domain', 'domains.yaml'), ('playlist', 'playlists.yaml'), ('repo', 'repos.yaml')): - register_ownership(_config, Collection, row=lambda collection, kind=_kind: collection.kind == kind) + register_ownership(_config, Collection, columns=COLLECTION_CONFIG_COLUMNS, + row=lambda collection, kind=_kind: collection.kind == kind) # Only a playlist's items are saved (author and subject Collections have items too). register_ownership('playlists.yaml', CollectionItem, row=lambda item: 'playlist' in collection_kinds(item)) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index e3df6e44..bd51c21f 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -2310,11 +2310,14 @@ class DownloadManagerConfig(ConfigFile): DOWNLOAD_MANAGER_CONFIG: DownloadManagerConfig = DownloadManagerConfig() -register_ownership(DownloadManagerConfig.file_name, Download) -# channels.yaml and repos.yaml each save their own Collections' Downloads (the URL and how often it runs). Only -# the config of that Collection's kind owns one: counting another config's change would make that config's next -# import save instead of import. -DOWNLOAD_COLLECTION_CONFIGS = (('channel', 'channels.yaml'), ('repo', 'repos.yaml')) +# Each config owns only the columns it saves: counting another change would make its next import save instead of +# import. (`attempts`, `error`, ... change on every attempt.) +register_ownership(DownloadManagerConfig.file_name, Download, + columns={'destination', 'downloader', 'frequency', 'last_successful_download', 'next_download', + 'settings', 'status', 'sub_downloader', 'tag_names', 'url'}) +# channels.yaml, domains.yaml and repos.yaml each save their own Collections' Downloads (the URL and how often it +# runs). Only the config of that Collection's kind owns one. +DOWNLOAD_COLLECTION_CONFIGS = (('channel', 'channels.yaml'), ('domain', 'domains.yaml'), ('repo', 'repos.yaml')) for _kind, _config in DOWNLOAD_COLLECTION_CONFIGS: register_ownership(_config, Download, columns={'url', 'frequency', 'collection_id'}, row=lambda download, kind=_kind: kind in collection_kinds(download)) diff --git a/wrolpi/test/test_config_version.py b/wrolpi/test/test_config_version.py index d8bf9bc7..b979d4a4 100644 --- a/wrolpi/test/test_config_version.py +++ b/wrolpi/test/test_config_version.py @@ -4,6 +4,7 @@ import yaml from wrolpi.collections.models import Collection from wrolpi.config_version import get_config_version, set_config_version +from wrolpi.dates import now from wrolpi.files.models import FileGroup from wrolpi.tags import Tag, get_tags_config, import_tags_config @@ -276,3 +277,60 @@ def test_only_playlist_items_count_for_playlists(test_session): playlist.add_url(test_session, '/map?x=1', title='Spot') test_session.commit() assert db_version(test_session, 'playlists.yaml') == 2 + + +def test_unsaved_columns_do_not_count(test_session): + """Columns a config does not save (a fetch's commit, a refresh's counts, a download attempt) do not count: + counting them would make that config's next import save instead of import.""" + from modules.repos.models import Repository + from modules.videos.models import Channel + from wrolpi.downloader import Download + + channel_collection = Collection(name='Channel', kind='channel') + repo_collection = Collection(name='Repo', kind='repo') + test_session.add_all([channel_collection, repo_collection]) + test_session.flush() + channel = Channel(collection_id=channel_collection.id) + repo = Repository(url='https://example.com/repo.git', collection_id=repo_collection.id) + download = Download(url='https://example.com/channel', frequency=86400, collection_id=channel_collection.id) + test_session.add_all([channel, repo, download]) + test_session.commit() + names = ('channels.yaml', 'repos.yaml', 'download_manager.yaml') + for name in names: + set_config_version(test_session, name, 1) + test_session.commit() + + repo.head_sha = 'abc123' + repo.last_fetch = now() + channel.video_count = 10 + channel.refreshed = True + channel_collection.item_count = 10 + repo_collection.total_size = 1000 + download.attempts = 1 + download.error = 'Failed' + download.last_download_attempt = now() + test_session.commit() + assert {i: db_version(test_session, i) for i in names} == dict.fromkeys(names, 1) + + # Saved columns still count. + repo.branch = 'dev' + channel.generate_posters = True + download.status = 'failed' + test_session.commit() + assert {i: db_version(test_session, i) for i in names} == dict.fromkeys(names, 2) + + +def test_domain_downloads_count_for_domains(test_session): + """domains.yaml saves each domain's Downloads (url, frequency).""" + from wrolpi.downloader import Download + + domain = Collection(name='example.com', kind='domain') + test_session.add(domain) + test_session.commit() + set_config_version(test_session, 'domains.yaml', 1) + test_session.commit() + + test_session.add(Download(url='https://example.com/feed', frequency=86400, collection_id=domain.id)) + test_session.commit() + + assert db_version(test_session, 'domains.yaml') == 2