Automatically backing up config files each day in config/backup directory.

modules/archive/__init__.py

@@ -283,6 +283,7 @@ async def archive_modeler():
                         except InvalidArchive:
                             # It was not a real Archive.  Many HTML files will not be an Archive.
                             file_group.indexed = False
+                            invalid_archives.add(file_group.id)
 
             session.commit()
 

wrolpi/common.py

@@ -9,6 +9,7 @@ import multiprocessing
 import os
 import pathlib
 import re
+import shutil
 import string
 import sys
 import tempfile
@@ -290,6 +291,15 @@ class ConfigFile:
             with config_file.open('rt') as fh:
                 self._config.update(yaml.load(fh, Loader=yaml.Loader))
 
+    def _get_backup_filename(self):
+        """Returns the path for the backup file for today."""
+        # TODO what if the RPi clock is not working?  Change this so the "version" of the config is used each day.
+        path = get_media_directory() / f'config/backup/{self.file_name}'
+        date_str = now().strftime('%Y%m%d')
+        name = f'{path.stem}-{date_str}{path.suffix}'
+        path = path.with_name(name)
+        return path
+
     def save(self):
         """
         Write this config to its file.
@@ -310,15 +320,22 @@ class ConfigFile:
             if not config_file.parent.is_dir():
                 config_file.parent.mkdir()
 
+            backup_file = self._get_backup_filename()
+
             # Read the existing config, replace all values, then save.
             if config_file.is_file():
                 with config_file.open('rt') as fh:
                     config = yaml.load(fh, Loader=yaml.Loader)
+                # Copy the existing config to the backup directory.  One for each day.
+                if not backup_file.parent.exists():
+                    backup_file.parent.mkdir(parents=True)
+                shutil.copy(config_file, backup_file)
             else:
                 # Config file does not yet exist.
                 config = dict()
 
             config.update({k: v for k, v in self._config.items() if v is not None})
+            logger.warning(f'Saving config: {config_file}')
             with config_file.open('wt') as fh:
                 yaml.dump(config, fh, width=self.width)
         finally:

wrolpi/files/test/test_api.py

@@ -655,7 +655,9 @@ async def test_ignore_directory(test_session, test_async_client, test_directory,
     await lib.refresh_files()
 
     files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
-    assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'baz.txt', 'wrolpi.yaml'}
+    # Ignore configs.
+    files = {i for i in files if not i.primary_path.name.endswith('.yaml')}
+    assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'baz.txt'}
 
     # Cannot ignore special directories.
     content = dict(path=str(test_directory / 'videos'))

wrolpi/tags.py

@@ -158,6 +158,7 @@ class TagsConfig(ConfigFile):
                 tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(),
             ]
             tag_files.append(value)
+        logger.debug(f'Got {len(tag_files)} tag files for config.')
 
         from modules.zim.models import Zim, TagZimEntry
         results = session.query(Tag, Zim, TagZimEntry) \
@@ -176,6 +177,7 @@ class TagsConfig(ConfigFile):
                 tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(),
             ]
             tag_zims.append(value)
+        logger.debug(f'Got {len(tag_zims)} tag zims for config.')
 
         # Write to the config.
         self.update({
@@ -481,3 +483,11 @@ def tag_names_to_zim_sub_select(tag_names: List[str], zim_id: int = None) -> Tup
             HAVING array_agg(t.name)::TEXT[] @> %(tag_names)s::TEXT[]
         '''
     return stmt, params
+
+
+if __name__ == '__main__':
+    from wrolpi.db import get_db_session
+
+    if input('Save Tags to config file? (y/n) ').strip().lower() == 'y':
+        with get_db_session() as session:
+            get_tags_config().save_tags(session)

wrolpi/test/test_common.py

@@ -4,6 +4,7 @@ import json
 import multiprocessing
 import os
 import pathlib
+import re
 import tempfile
 from datetime import date, datetime
 from decimal import Decimal
@@ -797,3 +798,24 @@ def test_find_file(test_directory, make_files_structure):
     assert common.find_file(test_directory, 'baz.txt', 2) == baz
 
     assert common.find_file(test_directory, 'does not exist', 100) is None
+
+
+def test_config_backup(test_config, test_directory):
+    """Configs have a backup each day."""
+    config = common.get_wrolpi_config()
+
+    # No backups yet.
+    config.save()
+    assert (test_directory / 'config').is_dir()
+    assert not (test_directory / 'config/backup').is_dir()
+    assert (test_directory / 'config/wrolpi.yaml').is_file()
+
+    # Backups directory is created.  New backup config is saved.
+    config.save()
+    assert (test_directory / 'config').is_dir()
+    assert (test_directory / 'config/backup').is_dir()
+    assert (test_directory / 'config/wrolpi.yaml').is_file()
+
+    backup_file, = list((test_directory / 'config/backup').glob('*'))
+    assert re.match(r'wrolpi-\d{8}\.yaml', backup_file.name), \
+        f'{backup_file.name} is not in the expected format'