diff --git a/.circleci/config.yml b/.circleci/config.yml index 87549ac9..bde6e703 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,11 @@ jobs: paths: - "venv" - run: + name: Run Alembic Migrations command: ./venv/bin/python ./main.py db upgrade + - run: + name: Check for Missing Migrations + command: ./venv/bin/python ./main.py db check api-tests-3-12: docker: - image: cimg/python:3.12 diff --git a/alembic/env.py b/alembic/env.py index 751d50af..ac05474b 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -15,9 +15,20 @@ fileConfig(config.config_file_name) # add your model's MetaData object here # for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = None +from wrolpi.common import Base +# Import all models so they are registered with the Base metadata +from wrolpi.files.models import FileGroup, Directory # noqa: F401 +from wrolpi.files.ebooks import EBook # noqa: F401 +from wrolpi.tags import Tag # noqa: F401 +from wrolpi.flags import WROLPiFlag # noqa: F401 +from wrolpi.downloader import Download # noqa: F401 +from wrolpi.collections.models import Collection, CollectionItem # noqa: F401 +from modules.videos.models import Video, Channel # noqa: F401 +from modules.archive.models import Archive # noqa: F401 +from modules.zim.models import Zim, TagZimEntry, ZimSubscription # noqa: F401 +from modules.map.models import MapFile # noqa: F401 +from modules.inventory.models import Item, Inventory # noqa: F401 +target_metadata = Base.metadata # other values from the config, defined by the needs of env.py, diff --git a/alembic/versions/f38cfe4b5cb2_collections_feature.py b/alembic/versions/f38cfe4b5cb2_collections_feature.py new file mode 100644 index 00000000..39d0ed9a --- /dev/null +++ b/alembic/versions/f38cfe4b5cb2_collections_feature.py @@ -0,0 +1,68 @@ +"""collections feature + +Revision ID: f38cfe4b5cb2 +Revises: add_unique_collection_name_kind +Create Date: 2025-11-29 16:51:34.222412 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'f38cfe4b5cb2' +down_revision = 'add_unique_collection_name_kind' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('archive', 'collection_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('file_group', 'mimetype', + existing_type=sa.TEXT(), + nullable=True) + op.alter_column('tag_file', 'tag_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('tag_file', 'file_group_id', + existing_type=sa.BIGINT(), + nullable=False) + op.alter_column('tag_zim', 'tag_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('tag_zim', 'zim_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_index('video_upload_date_idx', table_name='video') + op.drop_column('video', 'duration') + op.drop_column('video', 'upload_date') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('video', sa.Column('upload_date', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True)) + op.add_column('video', sa.Column('duration', sa.INTEGER(), autoincrement=False, nullable=True)) + op.create_index('video_upload_date_idx', 'video', ['upload_date'], unique=False) + op.alter_column('tag_zim', 'zim_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('tag_zim', 'tag_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('tag_file', 'file_group_id', + existing_type=sa.BIGINT(), + nullable=True) + op.alter_column('tag_file', 'tag_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('archive', 'collection_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('file_group', 'mimetype', + existing_type=sa.TEXT(), + nullable=False) + # ### end Alembic commands ### diff --git a/main.py b/main.py index 28e13bab..8abff9af 100755 --- a/main.py +++ b/main.py @@ -32,7 +32,7 @@ logger = logger.getChild('wrolpi-main') def db_main(args): """ - Handle database migrations. This uses Alembic, supported commands are "upgrade" and "downgrade". + Handle database migrations. This uses Alembic, supported commands are "upgrade", "downgrade", and "check". """ from alembic.config import Config from alembic import command @@ -48,6 +48,11 @@ def db_main(args): command.upgrade(config, 'head') elif args.command == 'downgrade': command.downgrade(config, '-1') + elif args.command == 'check': + command.check(config) + elif args.command == 'revision': + message = getattr(args, 'message', None) or 'auto migration' + command.revision(config, message=message, autogenerate=True) else: print(f'Unknown DB command: {args.command}') return 2 @@ -100,7 +105,8 @@ def main(): # DB Parser for running Alembic migrations db_parser = sub_commands.add_parser('db') - db_parser.add_argument('command', help='Supported commands: upgrade, downgrade') + db_parser.add_argument('command', help='Supported commands: upgrade, downgrade, check, revision') + db_parser.add_argument('-m', '--message', help='Migration message (for revision command)') args = parser.parse_args() diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index b2702595..974495ed 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -190,6 +190,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive: # Set collection_id BEFORE adding to session to avoid autoflush constraint violation from modules.archive.lib import get_or_create_domain_collection + if file_group.url: # URL is already set on the FileGroup collection = get_or_create_domain_collection(session, file_group.url) @@ -202,12 +203,8 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive: collection = get_or_create_domain_collection(session, url) archive.collection_id = collection.id if collection else None except (RuntimeError, ValueError) as e: - # Could not extract URL from singlefile - if not PYTEST: - # In production, archives must have a URL/collection - raise InvalidArchive(f'Archive has no URL and could not extract from singlefile: {e}') from e - # In tests, allow archives without collections (factory will set it later) - logger.debug(f'Could not extract URL from singlefile (test mode): {e}') + # Could not extract URL from singlefile - archive will have no collection + logger.debug(f'Could not extract URL from singlefile: {e}') archive.collection_id = None session.add(archive) diff --git a/modules/inventory/models.py b/modules/inventory/models.py index c8289204..6b481d9b 100644 --- a/modules/inventory/models.py +++ b/modules/inventory/models.py @@ -23,7 +23,7 @@ class Item(Base, ModelHelper): subcategory = Column(String) unit = Column(String) - inventory_id = Column(Integer, ForeignKey('inventory.id')) + inventory_id = Column(Integer, ForeignKey('inventory.id', ondelete='CASCADE')) inventory = relationship('Inventory', primaryjoin="Item.inventory_id==Inventory.id", back_populates='items') def __repr__(self): @@ -35,7 +35,7 @@ class Inventory(Base, ModelHelper): __tablename__ = 'inventory' id = Column(Integer, primary_key=True) - name = Column(String, unique=True) + name = Column(String, unique=True, nullable=False) viewed_at = Column(TZDateTime) created_at = Column(TZDateTime, default=now) deleted_at = Column(TZDateTime) @@ -61,6 +61,3 @@ class Inventory(Base, ModelHelper): return session.query(Inventory).filter_by(name=name).one() -class InventoriesVersion(Base, ModelHelper): - __tablename__ = 'inventories_version' - version = Column(Integer, primary_key=True) diff --git a/modules/map/models.py b/modules/map/models.py index 356983b2..9462c299 100644 --- a/modules/map/models.py +++ b/modules/map/models.py @@ -10,8 +10,8 @@ class MapFile(Base, ModelHelper): __tablename__ = 'map_file' id = Column(Integer, primary_key=True) - path = Column(MediaPathType, unique=True, nullable=False) - imported = Column(Boolean, default=False, nullable=False) + path = Column(MediaPathType, nullable=False) + imported = Column(Boolean, default=False) size = Column(BigInteger) def __repr__(self): diff --git a/modules/videos/models.py b/modules/videos/models.py index a2a197fc..b4a25b5c 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -2,7 +2,7 @@ import json import pathlib from typing import Optional, Dict, List -from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger +from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger, Index, text from sqlalchemy.orm import relationship, Session, deferred from sqlalchemy.orm.collections import InstrumentedList @@ -28,6 +28,11 @@ __all__ = ['Video', 'Channel'] class Video(ModelHelper, Base): __tablename__ = 'video' + __table_args__ = ( + Index('video_channel_id_idx', 'channel_id'), + Index('video_source_id_idx', 'source_id'), + Index('video_view_count_idx', 'view_count'), + ) id = Column(Integer, primary_key=True) source_id = Column(String) # The id from yt-dlp @@ -530,9 +535,16 @@ class Video(ModelHelper, Base): class Channel(ModelHelper, Base): __tablename__ = 'channel' + __table_args__ = ( + Index('channel_minimum_frequency_idx', 'minimum_frequency'), + Index('channel_source_id', 'source_id'), + Index('channel_total_size_idx', 'total_size'), + Index('channel_video_count_idx', 'video_count'), + Index('channel_url_key', 'url', unique=True, postgresql_where=text('url IS NOT NULL')), + ) id = Column(Integer, primary_key=True) # name and directory are stored in Collection, accessed via properties - url = Column(String, unique=True) # will only be downloaded if related Download exists. + url = Column(String) # will only be downloaded if related Download exists. Partial unique index on non-NULL values. generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow. calculate_duration = Column(Boolean, default=True) # use ffmpeg to extract duration (slower than info json). download_missing_data = Column(Boolean, default=True) # fetch missing data like `source_id` and video comments. @@ -540,8 +552,8 @@ class Channel(ModelHelper, Base): refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed. # Columns updated by triggers - video_count = Column(Integer, default=0) # update_channel_video_count - total_size = Column(Integer, default=0) # update_channel_size + video_count = Column(Integer, default=0, nullable=False) # update_channel_video_count + total_size = Column(BigInteger, default=0, nullable=False) # update_channel_size minimum_frequency = Column(Integer) # update_channel_minimum_frequency info_json = deferred(Column(JSON)) diff --git a/modules/zim/models.py b/modules/zim/models.py index 80d74b2f..e1126ea2 100644 --- a/modules/zim/models.py +++ b/modules/zim/models.py @@ -6,7 +6,7 @@ from datetime import datetime from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional, Set from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher -from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean +from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint from sqlalchemy.orm import relationship, Session from sqlalchemy.orm.exc import NoResultFound # noqa @@ -67,12 +67,15 @@ class Zim(Base, ModelHelper): """ __tablename__ = 'zim' + __table_args__ = ( + UniqueConstraint('path', name='zim_path_key'), + ) id = Column(Integer, primary_key=True) path: pathlib.Path = Column(MediaPathType, nullable=False) - file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False) + file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False) file_group: FileGroup = relationship('FileGroup') - auto_search = Column(Boolean, nullable=False, default=True) + auto_search = Column(Boolean, default=True) def __repr__(self): return f'' @@ -353,8 +356,11 @@ class Zims: class TagZimEntry(Base): __tablename__ = 'tag_zim' + __table_args__ = ( + UniqueConstraint('tag_id', 'zim_id', 'zim_entry', name='tag_zim_tag_id_zim_id_zim_entry_key'), + ) - tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True) + tag_id = Column(Integer, ForeignKey('tag.id'), primary_key=True) tag: Tag = relationship('Tag') zim_id = Column(Integer, ForeignKey('zim.id', ondelete='CASCADE'), primary_key=True) zim: Zim = relationship('Zim') diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index f6a0b1ca..4814262f 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -161,10 +161,9 @@ async def test_zim_tags_config(async_client, test_session, test_directory, test_ ['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'], ] - # Delete all Tags so they are recreated. - test_session.query(tags.Tag).delete() - # Delete all TagFileEntry(s), import them again. + # Delete all TagZimEntry(s) first (due to FK constraint), then Tags so they are recreated. test_session.query(TagZimEntry).delete() + test_session.query(tags.Tag).delete() test_session.commit() tags.import_tags_config() diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index 31e17a51..20c6bdab 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -1,7 +1,7 @@ import pathlib from typing import Optional, List -from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func +from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func, BigInteger from sqlalchemy.orm import relationship, Session from sqlalchemy.orm.collections import InstrumentedList @@ -67,6 +67,8 @@ class Collection(ModelHelper, Base): UniqueConstraint('directory', name='uq_collection_directory'), UniqueConstraint('name', 'kind', name='uq_collection_name_kind'), Index('idx_collection_kind', 'kind'), + Index('idx_collection_item_count', 'item_count'), + Index('idx_collection_total_size', 'total_size'), ) id = Column(Integer, primary_key=True) @@ -80,14 +82,14 @@ class Collection(ModelHelper, Base): directory: Optional[pathlib.Path] = Column(MediaPathType, nullable=True) # Optional tag relationship (similar to Channel) - tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE')) + tag_id = Column(Integer, ForeignKey('tag.id')) tag = relationship('Tag', primaryjoin='Collection.tag_id==Tag.id') - created_date = Column(DateTime, server_default=func.now()) + created_date = Column(DateTime, server_default=func.now(), nullable=False) # Columns updated by triggers (similar to Channel) - item_count = Column(Integer, default=0) - total_size = Column(Integer, default=0) + item_count = Column(Integer, default=0, nullable=False) + total_size = Column(BigInteger, default=0, nullable=False) # Relationship to items (ordered) items: InstrumentedList = relationship( @@ -790,7 +792,7 @@ class CollectionItem(ModelHelper, Base): position = Column(Integer, nullable=False, default=0) # When this item was added - added_date = Column(DateTime, server_default=func.now()) + added_date = Column(DateTime, server_default=func.now(), nullable=False) # Relationships collection = relationship('Collection', back_populates='items') @@ -798,8 +800,10 @@ class CollectionItem(ModelHelper, Base): # Indexes for performance __table_args__ = ( + Index('idx_collection_item_collection_id', 'collection_id'), Index('idx_collection_item_collection_position', 'collection_id', 'position'), - Index('idx_collection_item_file_group', 'file_group_id'), + Index('idx_collection_item_file_group_id', 'file_group_id'), + Index('idx_collection_item_position', 'position'), UniqueConstraint('collection_id', 'file_group_id', name='uq_collection_file_group'), ) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 73f3957f..f58f422d 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -20,7 +20,7 @@ from urllib.parse import urlparse import feedparser import pytz from feedparser import FeedParserDict -from sqlalchemy import Column, Integer, String, Text, ForeignKey, ARRAY +from sqlalchemy import Column, Integer, String, Text, ForeignKey, ARRAY, Index from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Session, relationship from sqlalchemy.sql import Delete @@ -100,6 +100,9 @@ class DownloadStatus(str, Enum): class Download(ModelHelper, Base): # noqa """Model that is used to schedule downloads.""" __tablename__ = 'download' # noqa + __table_args__ = ( + Index('idx_download_collection_id', 'collection_id'), + ) id = Column(Integer, primary_key=True) url = Column(String, nullable=False, unique=True) diff --git a/wrolpi/files/ebooks.py b/wrolpi/files/ebooks.py index 6d9a2b23..43688bf7 100644 --- a/wrolpi/files/ebooks.py +++ b/wrolpi/files/ebooks.py @@ -4,7 +4,7 @@ import logging import pathlib from typing import List, Optional, Tuple -from sqlalchemy import Column, Integer, BigInteger, ForeignKey, or_, and_ +from sqlalchemy import Column, Integer, BigInteger, ForeignKey, or_, and_, Index from sqlalchemy.orm import relationship, Session from wrolpi.common import ModelHelper, Base, register_modeler, get_html_soup @@ -150,6 +150,9 @@ def discover_calibre_cover(ebook_path: pathlib.Path): class EBook(ModelHelper, Base): __tablename__ = 'ebook' + __table_args__ = ( + Index('ebook_size_idx', 'size'), + ) id = Column(Integer, primary_key=True) size: int = Column(Integer) diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 7eb2fda2..533aeb81 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -6,7 +6,7 @@ from datetime import datetime from functools import singledispatchmethod from typing import List, Type, Optional, Iterable -from sqlalchemy import Column, String, Computed, BigInteger, Boolean, event +from sqlalchemy import Column, String, Computed, BigInteger, Boolean, event, Index from sqlalchemy import types from sqlalchemy.orm import deferred, relationship, Session @@ -66,15 +66,30 @@ class FancyJSON(types.TypeDecorator): class FileGroup(ModelHelper, Base): __tablename__ = 'file_group' + __table_args__ = ( + Index('file_group_author_idx', 'author'), + Index('file_group_download_datetime_idx', 'download_datetime'), + Index('file_group_effective_datetime_idx', 'effective_datetime'), + Index('file_group_length_idx', 'length'), + Index('file_group_mimetype_idx', 'mimetype'), + Index('file_group_model_idx', 'model'), + Index('file_group_modification_datetime_idx', 'modification_datetime'), + Index('file_group_published_datetime_idx', 'published_datetime'), + Index('file_group_published_modified_datetime_idx', 'published_modified_datetime'), + Index('file_group_size_ix', 'size'), + Index('file_group_textsearch_idx', 'textsearch'), + Index('file_group_url_idx', 'url'), + Index('file_group_viewed_idx', 'viewed'), + ) id: int = Column(BigInteger, primary_key=True) author = Column(String) # name of the author, maybe even a URL censored = Column(Boolean) # the file is no longer available for download data = Column(FancyJSON) # populated by the modeler download_datetime = Column(TZDateTime) # the date WROLPi downloaded this file. - files = Column(FancyJSON, nullable=False) # populated during discovery + files = Column(FancyJSON) # populated during discovery idempotency = Column(TZDateTime) # used to track which files need to be deleted during refresh - indexed = Column(Boolean, default=lambda: False, nullable=False) # wrolpi.files.lib.apply_indexers + indexed = Column(Boolean, default=lambda: False) # wrolpi.files.lib.apply_indexers length = Column(BigInteger) # video duration, article words, etc. mimetype = Column(String) # wrolpi.files.lib.get_mimetype model = Column(String) # "video", "archive", "ebook", etc. @@ -550,6 +565,9 @@ def update_effective_datetime(mapper, connection, target): class Directory(ModelHelper, Base): """A representation of a file directory in the media directory.""" __tablename__ = 'directory' + __table_args__ = ( + Index('directory_name_idx', 'name'), + ) path: pathlib.Path = Column(MediaPathType, primary_key=True) name: str = Column(String, nullable=False) diff --git a/wrolpi/tags.py b/wrolpi/tags.py index 36c9d7c4..08a47e12 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -6,7 +6,7 @@ from typing import List, Dict, Tuple, Optional import cachetools from cachetools.keys import hashkey -from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger, event +from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger, event, UniqueConstraint from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import relationship, Session @@ -30,9 +30,12 @@ def get_tags_directory() -> pathlib.Path: class TagFile(ModelHelper, Base): __tablename__ = 'tag_file' + __table_args__ = ( + UniqueConstraint('tag_id', 'file_group_id', name='tag_file_tag_id_file_group_id_key'), + ) created_at: datetime = Column(TZDateTime, default=dates.now) - tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True) + tag_id = Column(Integer, ForeignKey('tag.id'), primary_key=True) tag = relationship('Tag', back_populates='tag_files') file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), primary_key=True) file_group = relationship('FileGroup', back_populates='tag_files')