mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
Converting Tag methods to use tag id or name.
This commit is contained in:
parent
2ad363281f
commit
c839acb270
22 changed files with 379 additions and 163 deletions
|
|
@ -27,7 +27,7 @@ PRETTY_NAME = 'Archive'
|
|||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['ArchiveDownloader', 'archive_downloader']
|
||||
__all__ = ['ArchiveDownloader', 'archive_downloader', 'model_archive']
|
||||
|
||||
|
||||
class ArchiveDownloader(Downloader, ABC):
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from typing import Iterable, List, Optional, Union
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
import pytz
|
||||
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
|
||||
|
|
@ -59,6 +59,20 @@ class Archive(Base, ModelHelper):
|
|||
raise UnknownArchive(f'Cannot find Archive with id {id_}')
|
||||
return archive
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_path(path: pathlib.Path | str, session: Session = None) -> Optional['Archive']:
|
||||
archive = session.query(Archive).join(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none()
|
||||
return archive
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def find_by_path(path: pathlib.Path | str, session: Session = None) -> Optional['Archive']:
|
||||
archive = Archive.get_by_path(path, session)
|
||||
if archive:
|
||||
return archive
|
||||
raise UnknownArchive(f'Cannot find Archive with path: {path}')
|
||||
|
||||
def my_paths(self, *mimetypes: str) -> List[pathlib.Path]:
|
||||
return self.file_group.my_paths(*mimetypes)
|
||||
|
||||
|
|
@ -285,9 +299,8 @@ class Archive(Base, ModelHelper):
|
|||
archive = model_archive(file_group, session)
|
||||
return archive
|
||||
|
||||
def add_tag(self, tag_or_tag_name: Union[Tag, str]) -> TagFile:
|
||||
tag = Tag.get_by_name(tag_or_tag_name) if isinstance(tag_or_tag_name, str) else tag_or_tag_name
|
||||
return self.file_group.add_tag(tag)
|
||||
def add_tag(self, tag_id_or_name: int | str) -> TagFile:
|
||||
return self.file_group.add_tag(tag_id_or_name)
|
||||
|
||||
|
||||
class Domain(Base, ModelHelper):
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from modules.videos.models import Channel, Video
|
|||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.downloader import DownloadFrequency, DownloadManager
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import Tag
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import json
|
||||
import pathlib
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List, Union
|
||||
from typing import Optional, Dict, List
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger
|
||||
from sqlalchemy.orm import relationship, Session, deferred
|
||||
|
|
@ -224,7 +224,7 @@ class Video(ModelHelper, Base):
|
|||
try:
|
||||
validate_video(self, self.channel.generate_posters if self.channel else False)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to validate video {self}', exc_info=e)
|
||||
logger.error(f'Failed to validate video {self}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
|
|
@ -339,9 +339,10 @@ class Video(ModelHelper, Base):
|
|||
raise UnknownVideo(f'Cannot find Video with id {id_}')
|
||||
return video
|
||||
|
||||
def add_tag(self, tag_id: int, session: Session = None) -> TagFile:
|
||||
def add_tag(self, tag_id_or_name: int, session: Session = None) -> TagFile:
|
||||
session = session or Session.object_session(self)
|
||||
return self.file_group.add_tag(tag_id, session=session)
|
||||
tag = Tag.find_by_name(tag_id_or_name) if isinstance(tag_id_or_name, str) else Tag.find_by_id(tag_id_or_name)
|
||||
return self.file_group.add_tag(tag.id, session=session)
|
||||
|
||||
async def get_ffprobe_json(self) -> dict:
|
||||
"""Return the ffprobe json object if previously stored.
|
||||
|
|
|
|||
|
|
@ -225,13 +225,13 @@ def test_search_videos(test_session, video_factory, assert_video_search, simple_
|
|||
vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
|
||||
vid1.add_tag(tag1)
|
||||
vid1.add_tag(tag1.id)
|
||||
|
||||
# vid2 has two tags
|
||||
vid2.add_tag(tag1)
|
||||
vid2.add_tag(tag2)
|
||||
vid2.add_tag(tag1.id)
|
||||
vid2.add_tag(tag2.id)
|
||||
|
||||
vid3.add_tag(tag2)
|
||||
vid3.add_tag(tag2.id)
|
||||
|
||||
test_session.commit()
|
||||
assert test_session.query(Video).count() == 3
|
||||
|
|
|
|||
|
|
@ -45,11 +45,12 @@ def test_delete_video(test_session, video_factory):
|
|||
assert test_session.query(FileGroup).count() == 0, 'Video files were not deleted.'
|
||||
|
||||
|
||||
def test_delete_video_with_tag(test_session, video_factory, tag_factory):
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_video_with_tag(test_async_client, test_session, video_factory, tag_factory):
|
||||
"""You cannot delete a video with a tag."""
|
||||
video = video_factory(with_video_file=True, with_poster_ext='png')
|
||||
tag = tag_factory()
|
||||
video.add_tag(tag)
|
||||
video.add_tag(tag.name)
|
||||
test_session.commit()
|
||||
|
||||
with pytest.raises(FileGroupIsTagged):
|
||||
|
|
|
|||
|
|
@ -151,7 +151,7 @@ def headline_zim(search_str: str, zim_id: int, tag_names: List[str] = None, offs
|
|||
# Remove duplicate entries (disambiguation or redirect).
|
||||
results = get_unique_paths(*results, *search_results)
|
||||
results = results[:limit]
|
||||
entries = [zim.get_entry(i) for i in results]
|
||||
entries = [zim.find_entry(i) for i in results]
|
||||
estimate = zim.estimate(search_str)
|
||||
else:
|
||||
raise RuntimeError('Must provide search_str or tag_names.')
|
||||
|
|
@ -238,7 +238,7 @@ def get_zim(zim_id: int, session: Session = None) -> Zim:
|
|||
def get_entry(path: str, zim_id: int) -> Entry:
|
||||
with get_db_session() as session:
|
||||
zim = get_zim(zim_id, session=session)
|
||||
entry = zim.get_entry(path, throw=True)
|
||||
entry = zim.find_entry(path)
|
||||
return entry
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -163,61 +163,54 @@ class Zim(Base):
|
|||
results = list(suggestions.getResults(offset, limit))
|
||||
return results
|
||||
|
||||
def get_entry(self, path: str, throw: bool = True) -> Optional[Entry]:
|
||||
def get_entry(self, path: str) -> Optional[Entry]:
|
||||
"""Search this Zim file for the provided Entry at the `path`."""
|
||||
zim = self.get_zim()
|
||||
try:
|
||||
entry = zim.get_entry_by_path(path)
|
||||
except KeyError:
|
||||
if throw:
|
||||
raise UnknownZimEntry(f'Cannot find entry at {path=}')
|
||||
else:
|
||||
return None
|
||||
return None
|
||||
return entry
|
||||
|
||||
def get_entry_html(self, path: str) -> Optional[str]:
|
||||
"""Search this Zim file for the provided Entry at `path`, read its contents and return the HTML."""
|
||||
content = bytes(self.get_entry(path).get_item().content).decode('UTF-8')
|
||||
return content
|
||||
def find_entry(self, path: str) -> Entry:
|
||||
"""Search this Zim file for the provided Entry at the `path`.
|
||||
|
||||
def get_entry_text(self, path: str) -> Optional[str]:
|
||||
"""Search this Zim file for the provided Entry at `path`, read its contents and return the text in the HTML."""
|
||||
html = self.get_entry_html(path)
|
||||
content = extract_html_text(html)
|
||||
return content
|
||||
@raise UnknownZimEntry: if an entry at path does not exist."""
|
||||
if entry := self.get_entry(path):
|
||||
return entry
|
||||
|
||||
def tag_entry(self, tag_name: str, zim_entry: str) -> 'TagZimEntry':
|
||||
raise UnknownZimEntry(f'Cannot find entry at {path=}')
|
||||
|
||||
def tag_entry(self, tag_id_or_name: int | str, zim_entry: str) -> 'TagZimEntry':
|
||||
"""Create a TagZimEntry for this Zim at the provided entry (path) for the provided Tag."""
|
||||
session = Session.object_session(self)
|
||||
tag = Tag.get_by_name(tag_name, session=session)
|
||||
|
||||
# Ensure the entry exists.
|
||||
self.find_entry(zim_entry)
|
||||
|
||||
tag = Tag.get_by_name(tag_id_or_name, session) if isinstance(tag_id_or_name, str) \
|
||||
else Tag.get_by_id(tag_id_or_name, session)
|
||||
tag_zim_entry = TagZimEntry(tag=tag, zim=self, zim_entry=zim_entry)
|
||||
session.add(tag_zim_entry)
|
||||
tags.schedule_save()
|
||||
tags.save_tags_config()
|
||||
return tag_zim_entry
|
||||
|
||||
def untag_entry(self, tag_name: str, zim_entry: str):
|
||||
def untag_entry(self, tag_id_or_name: int | str, zim_entry: str):
|
||||
"""Removes any TagZimEntry of the Zim entry at the provided path for the provided Tag.
|
||||
|
||||
@raise UnknownZimTagEntry: No entry exists to remove."""
|
||||
session = Session.object_session(self)
|
||||
tag = Tag.get_by_name(tag_name, session=session)
|
||||
tag_zim_entry = session.query(TagZimEntry).filter_by(
|
||||
tag_id=tag.id,
|
||||
zim_id=self.id,
|
||||
zim_entry=zim_entry,
|
||||
).one_or_none()
|
||||
tag = Tag.get_by_name(tag_id_or_name, session) if isinstance(tag_id_or_name, str) \
|
||||
else Tag.get_by_id(tag_id_or_name, session)
|
||||
tag_zim_entry = TagZimEntry.get_by_primary_keys(tag.id, self.id, zim_entry, session)
|
||||
if tag_zim_entry:
|
||||
session.delete(tag_zim_entry)
|
||||
else:
|
||||
raise UnknownZimTagEntry(f'Could not find at {repr(str(zim_entry))}')
|
||||
tags.schedule_save()
|
||||
|
||||
@optional_session
|
||||
def entries_with_tags(self, tag_names: List[str], offset: int = 0, limit: int = 10, session: Session = None) \
|
||||
-> List[Entry]:
|
||||
"""Return Zim Entries tagged with the provided names."""
|
||||
tags_sub_select, params = tag_names_to_zim_sub_select(tag_names, zim_id=self.id)
|
||||
tags.save_tags_config()
|
||||
|
||||
@staticmethod
|
||||
def _entries_with_tags_process(limit: int, offset: int, params: dict, session: Session, tags_sub_select: str):
|
||||
with get_db_curs() as curs:
|
||||
curs.execute(tags_sub_select, params)
|
||||
zim_ids_entries: List[Tuple[int, int]] = list(curs.fetchall())
|
||||
|
|
@ -231,11 +224,31 @@ class Zim(Base):
|
|||
|
||||
entries = list()
|
||||
for tag_zim_entry, zim in tag_zim_entries:
|
||||
entry = zim.get_entry(tag_zim_entry.zim_entry)
|
||||
entry = zim.find_entry(tag_zim_entry.zim_entry)
|
||||
entries.append(entry)
|
||||
|
||||
return entries
|
||||
|
||||
@optional_session
|
||||
def entries_with_tags(*args, session: Session = None, **kwargs) -> List[Entry]:
|
||||
"""Return Zim Entries tagged with the provided names."""
|
||||
limit = int(kwargs.pop('limit', 10))
|
||||
offset = int(kwargs.pop('offset', 0))
|
||||
args = list(args)
|
||||
|
||||
zim: Zim | None = args.pop(0) if isinstance(args[0], Zim) else None
|
||||
zim_id = zim.id if zim else None
|
||||
tag_names: List[str] = args.pop(0)
|
||||
|
||||
if limit < 1:
|
||||
raise RuntimeError('Limit must be greater than zero')
|
||||
if args:
|
||||
raise RuntimeError('Extra unknown arguments')
|
||||
|
||||
tags_sub_select, params = tag_names_to_zim_sub_select(tag_names, zim_id=zim_id)
|
||||
entries = Zim._entries_with_tags_process(limit, offset, params, session, tags_sub_select)
|
||||
return entries
|
||||
|
||||
@functools.cached_property
|
||||
def all_entries(self) -> Set[EntrySummary]:
|
||||
"""
|
||||
|
|
@ -258,11 +271,6 @@ class Zim(Base):
|
|||
entries.add(summary)
|
||||
return entries
|
||||
|
||||
def parse_name(self):
|
||||
from modules.zim import lib
|
||||
name, date = lib.parse_name(self.path)
|
||||
return name, date
|
||||
|
||||
|
||||
class Zims:
|
||||
"""Convenience class which searches all known Zim files."""
|
||||
|
|
@ -317,7 +325,13 @@ class TagZimEntry(Base):
|
|||
zim = self.zim_id
|
||||
if self.zim:
|
||||
zim = get_relative_to_media_directory(self.zim.path)
|
||||
return f'<TagZimEntry {tag=} {zim=} zim_entry={self.zim_entry}>'
|
||||
return f'<TagZimEntry {tag=} zim={str(zim)} zim_entry={self.zim_entry}>'
|
||||
|
||||
@staticmethod
|
||||
def get_by_primary_keys(tag_id: int, zim_id: int, zim_entry: str, session: Session):
|
||||
return session.query(TagZimEntry) \
|
||||
.filter_by(tag_id=tag_id, zim_id=zim_id, zim_entry=zim_entry) \
|
||||
.one_or_none()
|
||||
|
||||
|
||||
class ZimSubscription(Base):
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import pytest
|
|||
from libzim import Entry
|
||||
|
||||
from modules.zim import lib
|
||||
from modules.zim.errors import UnknownZim
|
||||
from modules.zim.errors import UnknownZim, UnknownZimEntry
|
||||
from modules.zim.models import TagZimEntry, ZimSubscription
|
||||
from wrolpi import tags, flags
|
||||
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
|
||||
|
|
@ -49,6 +49,13 @@ async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
|
|||
assert entry.title == 'Homepage'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_bad_entry(test_async_client, test_session, test_zim, tag_factory):
|
||||
tag1 = tag_factory()
|
||||
with pytest.raises(UnknownZimEntry):
|
||||
test_zim.tag_entry(tag1.name, 'path does not exist')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, tag_factory):
|
||||
tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
|
||||
|
|
@ -61,6 +68,37 @@ async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, t
|
|||
== dict(one=['tag1', 'tag2'], two=['tag1'], home=[])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, tag_factory):
|
||||
tze = zim_factory('zim1').tag_entry(tag_factory().name, 'one')
|
||||
# Test TagZimEntry.__repr__
|
||||
assert repr(tze) == "<TagZimEntry tag='one' zim=zim1 zim_entry=one>"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_entries_with_tags(test_async_client, test_session, zim_factory, tag_factory):
|
||||
"""Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim."""
|
||||
tag1, tag2 = tag_factory(), tag_factory()
|
||||
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
|
||||
# Tag Zim1 with both tags, but Zim2 shares one of the tags and is returned when no Zim is passed.
|
||||
zim1.tag_entry(tag1.name, 'one')
|
||||
zim1.tag_entry(tag2.name, 'two')
|
||||
zim2.tag_entry(tag2.name, 'home')
|
||||
test_session.commit()
|
||||
|
||||
# Get entries from zim1 only.
|
||||
entries = zim1.entries_with_tags([tag1.name, ])
|
||||
assert len(entries) == 1
|
||||
assert entries[0].path == 'one'
|
||||
entries = zim1.entries_with_tags([tag2.name, ])
|
||||
assert len(entries) == 1
|
||||
assert entries[0].path == 'two'
|
||||
|
||||
# Get entries from any Zim.
|
||||
entries = lib.Zim.entries_with_tags([tag2.name, ])
|
||||
assert {i.path for i in entries} == {'two', 'home'}, 'Tag2 was applied to `two` and `home`'
|
||||
|
||||
|
||||
def test_get_unique_paths():
|
||||
paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear',
|
||||
'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue