Converting Tag methods to use tag id or name.

This commit is contained in:
Roland 2024-07-30 17:15:01 -06:00
parent 2ad363281f
commit c839acb270
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
22 changed files with 379 additions and 163 deletions

View file

@ -27,7 +27,7 @@ PRETTY_NAME = 'Archive'
logger = logger.getChild(__name__)
__all__ = ['ArchiveDownloader', 'archive_downloader']
__all__ = ['ArchiveDownloader', 'archive_downloader', 'model_archive']
class ArchiveDownloader(Downloader, ABC):

View file

@ -1,7 +1,7 @@
import json
import pathlib
import re
from typing import Iterable, List, Optional, Union
from typing import Iterable, List, Optional
import pytz
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
@ -59,6 +59,20 @@ class Archive(Base, ModelHelper):
raise UnknownArchive(f'Cannot find Archive with id {id_}')
return archive
@staticmethod
@optional_session
def get_by_path(path: pathlib.Path | str, session: Session = None) -> Optional['Archive']:
archive = session.query(Archive).join(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none()
return archive
@staticmethod
@optional_session
def find_by_path(path: pathlib.Path | str, session: Session = None) -> Optional['Archive']:
archive = Archive.get_by_path(path, session)
if archive:
return archive
raise UnknownArchive(f'Cannot find Archive with path: {path}')
def my_paths(self, *mimetypes: str) -> List[pathlib.Path]:
return self.file_group.my_paths(*mimetypes)
@ -285,9 +299,8 @@ class Archive(Base, ModelHelper):
archive = model_archive(file_group, session)
return archive
def add_tag(self, tag_or_tag_name: Union[Tag, str]) -> TagFile:
tag = Tag.get_by_name(tag_or_tag_name) if isinstance(tag_or_tag_name, str) else tag_or_tag_name
return self.file_group.add_tag(tag)
def add_tag(self, tag_id_or_name: int | str) -> TagFile:
return self.file_group.add_tag(tag_id_or_name)
class Domain(Base, ModelHelper):

View file

@ -16,6 +16,7 @@ from modules.videos.models import Channel, Video
from wrolpi.api_utils import api_app
from wrolpi.downloader import DownloadFrequency, DownloadManager
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag
from wrolpi.vars import PROJECT_DIR

View file

@ -1,7 +1,7 @@
import json
import pathlib
from pathlib import Path
from typing import Optional, Dict, List, Union
from typing import Optional, Dict, List
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger
from sqlalchemy.orm import relationship, Session, deferred
@ -224,7 +224,7 @@ class Video(ModelHelper, Base):
try:
validate_video(self, self.channel.generate_posters if self.channel else False)
except Exception as e:
logger.warning(f'Failed to validate video {self}', exc_info=e)
logger.error(f'Failed to validate video {self}', exc_info=e)
if PYTEST:
raise
@ -339,9 +339,10 @@ class Video(ModelHelper, Base):
raise UnknownVideo(f'Cannot find Video with id {id_}')
return video
def add_tag(self, tag_id: int, session: Session = None) -> TagFile:
def add_tag(self, tag_id_or_name: int, session: Session = None) -> TagFile:
session = session or Session.object_session(self)
return self.file_group.add_tag(tag_id, session=session)
tag = Tag.find_by_name(tag_id_or_name) if isinstance(tag_id_or_name, str) else Tag.find_by_id(tag_id_or_name)
return self.file_group.add_tag(tag.id, session=session)
async def get_ffprobe_json(self) -> dict:
"""Return the ffprobe json object if previously stored.

View file

@ -225,13 +225,13 @@ def test_search_videos(test_session, video_factory, assert_video_search, simple_
vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
tag1, tag2 = tag_factory(), tag_factory()
vid1.add_tag(tag1)
vid1.add_tag(tag1.id)
# vid2 has two tags
vid2.add_tag(tag1)
vid2.add_tag(tag2)
vid2.add_tag(tag1.id)
vid2.add_tag(tag2.id)
vid3.add_tag(tag2)
vid3.add_tag(tag2.id)
test_session.commit()
assert test_session.query(Video).count() == 3

View file

@ -45,11 +45,12 @@ def test_delete_video(test_session, video_factory):
assert test_session.query(FileGroup).count() == 0, 'Video files were not deleted.'
def test_delete_video_with_tag(test_session, video_factory, tag_factory):
@pytest.mark.asyncio
async def test_delete_video_with_tag(test_async_client, test_session, video_factory, tag_factory):
"""You cannot delete a video with a tag."""
video = video_factory(with_video_file=True, with_poster_ext='png')
tag = tag_factory()
video.add_tag(tag)
video.add_tag(tag.name)
test_session.commit()
with pytest.raises(FileGroupIsTagged):

View file

@ -151,7 +151,7 @@ def headline_zim(search_str: str, zim_id: int, tag_names: List[str] = None, offs
# Remove duplicate entries (disambiguation or redirect).
results = get_unique_paths(*results, *search_results)
results = results[:limit]
entries = [zim.get_entry(i) for i in results]
entries = [zim.find_entry(i) for i in results]
estimate = zim.estimate(search_str)
else:
raise RuntimeError('Must provide search_str or tag_names.')
@ -238,7 +238,7 @@ def get_zim(zim_id: int, session: Session = None) -> Zim:
def get_entry(path: str, zim_id: int) -> Entry:
with get_db_session() as session:
zim = get_zim(zim_id, session=session)
entry = zim.get_entry(path, throw=True)
entry = zim.find_entry(path)
return entry

View file

@ -163,61 +163,54 @@ class Zim(Base):
results = list(suggestions.getResults(offset, limit))
return results
def get_entry(self, path: str, throw: bool = True) -> Optional[Entry]:
def get_entry(self, path: str) -> Optional[Entry]:
"""Search this Zim file for the provided Entry at the `path`."""
zim = self.get_zim()
try:
entry = zim.get_entry_by_path(path)
except KeyError:
if throw:
raise UnknownZimEntry(f'Cannot find entry at {path=}')
else:
return None
return None
return entry
def get_entry_html(self, path: str) -> Optional[str]:
"""Search this Zim file for the provided Entry at `path`, read its contents and return the HTML."""
content = bytes(self.get_entry(path).get_item().content).decode('UTF-8')
return content
def find_entry(self, path: str) -> Entry:
"""Search this Zim file for the provided Entry at the `path`.
def get_entry_text(self, path: str) -> Optional[str]:
"""Search this Zim file for the provided Entry at `path`, read its contents and return the text in the HTML."""
html = self.get_entry_html(path)
content = extract_html_text(html)
return content
@raise UnknownZimEntry: if an entry at path does not exist."""
if entry := self.get_entry(path):
return entry
def tag_entry(self, tag_name: str, zim_entry: str) -> 'TagZimEntry':
raise UnknownZimEntry(f'Cannot find entry at {path=}')
def tag_entry(self, tag_id_or_name: int | str, zim_entry: str) -> 'TagZimEntry':
"""Create a TagZimEntry for this Zim at the provided entry (path) for the provided Tag."""
session = Session.object_session(self)
tag = Tag.get_by_name(tag_name, session=session)
# Ensure the entry exists.
self.find_entry(zim_entry)
tag = Tag.get_by_name(tag_id_or_name, session) if isinstance(tag_id_or_name, str) \
else Tag.get_by_id(tag_id_or_name, session)
tag_zim_entry = TagZimEntry(tag=tag, zim=self, zim_entry=zim_entry)
session.add(tag_zim_entry)
tags.schedule_save()
tags.save_tags_config()
return tag_zim_entry
def untag_entry(self, tag_name: str, zim_entry: str):
def untag_entry(self, tag_id_or_name: int | str, zim_entry: str):
"""Removes any TagZimEntry of the Zim entry at the provided path for the provided Tag.
@raise UnknownZimTagEntry: No entry exists to remove."""
session = Session.object_session(self)
tag = Tag.get_by_name(tag_name, session=session)
tag_zim_entry = session.query(TagZimEntry).filter_by(
tag_id=tag.id,
zim_id=self.id,
zim_entry=zim_entry,
).one_or_none()
tag = Tag.get_by_name(tag_id_or_name, session) if isinstance(tag_id_or_name, str) \
else Tag.get_by_id(tag_id_or_name, session)
tag_zim_entry = TagZimEntry.get_by_primary_keys(tag.id, self.id, zim_entry, session)
if tag_zim_entry:
session.delete(tag_zim_entry)
else:
raise UnknownZimTagEntry(f'Could not find at {repr(str(zim_entry))}')
tags.schedule_save()
@optional_session
def entries_with_tags(self, tag_names: List[str], offset: int = 0, limit: int = 10, session: Session = None) \
-> List[Entry]:
"""Return Zim Entries tagged with the provided names."""
tags_sub_select, params = tag_names_to_zim_sub_select(tag_names, zim_id=self.id)
tags.save_tags_config()
@staticmethod
def _entries_with_tags_process(limit: int, offset: int, params: dict, session: Session, tags_sub_select: str):
with get_db_curs() as curs:
curs.execute(tags_sub_select, params)
zim_ids_entries: List[Tuple[int, int]] = list(curs.fetchall())
@ -231,11 +224,31 @@ class Zim(Base):
entries = list()
for tag_zim_entry, zim in tag_zim_entries:
entry = zim.get_entry(tag_zim_entry.zim_entry)
entry = zim.find_entry(tag_zim_entry.zim_entry)
entries.append(entry)
return entries
@optional_session
def entries_with_tags(*args, session: Session = None, **kwargs) -> List[Entry]:
"""Return Zim Entries tagged with the provided names."""
limit = int(kwargs.pop('limit', 10))
offset = int(kwargs.pop('offset', 0))
args = list(args)
zim: Zim | None = args.pop(0) if isinstance(args[0], Zim) else None
zim_id = zim.id if zim else None
tag_names: List[str] = args.pop(0)
if limit < 1:
raise RuntimeError('Limit must be greater than zero')
if args:
raise RuntimeError('Extra unknown arguments')
tags_sub_select, params = tag_names_to_zim_sub_select(tag_names, zim_id=zim_id)
entries = Zim._entries_with_tags_process(limit, offset, params, session, tags_sub_select)
return entries
@functools.cached_property
def all_entries(self) -> Set[EntrySummary]:
"""
@ -258,11 +271,6 @@ class Zim(Base):
entries.add(summary)
return entries
def parse_name(self):
from modules.zim import lib
name, date = lib.parse_name(self.path)
return name, date
class Zims:
"""Convenience class which searches all known Zim files."""
@ -317,7 +325,13 @@ class TagZimEntry(Base):
zim = self.zim_id
if self.zim:
zim = get_relative_to_media_directory(self.zim.path)
return f'<TagZimEntry {tag=} {zim=} zim_entry={self.zim_entry}>'
return f'<TagZimEntry {tag=} zim={str(zim)} zim_entry={self.zim_entry}>'
@staticmethod
def get_by_primary_keys(tag_id: int, zim_id: int, zim_entry: str, session: Session):
return session.query(TagZimEntry) \
.filter_by(tag_id=tag_id, zim_id=zim_id, zim_entry=zim_entry) \
.one_or_none()
class ZimSubscription(Base):

View file

@ -5,7 +5,7 @@ import pytest
from libzim import Entry
from modules.zim import lib
from modules.zim.errors import UnknownZim
from modules.zim.errors import UnknownZim, UnknownZimEntry
from modules.zim.models import TagZimEntry, ZimSubscription
from wrolpi import tags, flags
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
@ -49,6 +49,13 @@ async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
assert entry.title == 'Homepage'
@pytest.mark.asyncio
async def test_zim_bad_entry(test_async_client, test_session, test_zim, tag_factory):
tag1 = tag_factory()
with pytest.raises(UnknownZimEntry):
test_zim.tag_entry(tag1.name, 'path does not exist')
@pytest.mark.asyncio
async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, tag_factory):
tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
@ -61,6 +68,37 @@ async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, t
== dict(one=['tag1', 'tag2'], two=['tag1'], home=[])
@pytest.mark.asyncio
async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, tag_factory):
tze = zim_factory('zim1').tag_entry(tag_factory().name, 'one')
# Test TagZimEntry.__repr__
assert repr(tze) == "<TagZimEntry tag='one' zim=zim1 zim_entry=one>"
@pytest.mark.asyncio
async def test_zim_entries_with_tags(test_async_client, test_session, zim_factory, tag_factory):
"""Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim."""
tag1, tag2 = tag_factory(), tag_factory()
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
# Tag Zim1 with both tags, but Zim2 shares one of the tags and is returned when no Zim is passed.
zim1.tag_entry(tag1.name, 'one')
zim1.tag_entry(tag2.name, 'two')
zim2.tag_entry(tag2.name, 'home')
test_session.commit()
# Get entries from zim1 only.
entries = zim1.entries_with_tags([tag1.name, ])
assert len(entries) == 1
assert entries[0].path == 'one'
entries = zim1.entries_with_tags([tag2.name, ])
assert len(entries) == 1
assert entries[0].path == 'two'
# Get entries from any Zim.
entries = lib.Zim.entries_with_tags([tag2.name, ])
assert {i.path for i in entries} == {'two', 'home'}, 'Tag2 was applied to `two` and `home`'
def test_get_unique_paths():
paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear',
'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear',