Replacing tag filtering with new tag_names_to_sub_select function which uses a single sub-select to filter FileGroups.

This commit is contained in:
Roland 2023-05-18 22:39:19 -06:00
parent fedbbe31d7
commit bc2ca0e512
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
5 changed files with 26 additions and 61 deletions

View file

@ -20,7 +20,7 @@ from wrolpi.common import get_media_directory, logger, extract_domain, chdir, es
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import UnknownArchive, InvalidOrderBy
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_sub_select
from wrolpi.vars import DOCKERIZED, PYTEST
logger = logger.getChild(__name__)
@ -416,10 +416,10 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
raise InvalidOrderBy(f'Invalid order by: {order}')
if tag_names:
where_, params_, join_ = tag_names_to_clauses(tag_names)
wheres.append(where_)
# Filter all FileGroups by those that have been tagged with the provided tag names.
tags_stmt, params_ = tag_names_to_sub_select(tag_names)
params.update(params_)
joins.append(join_)
wheres.append(f'fg.id = ANY({tags_stmt})')
if search_str and headline:
headline = ''',

View file

@ -7,7 +7,7 @@ from modules.videos.models import Video
from wrolpi.common import run_after, logger
from wrolpi.db import get_db_session, optional_session
from wrolpi.errors import UnknownVideo
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_sub_select
from ..lib import save_channels_config
logger.getChild(__name__)
@ -102,10 +102,10 @@ def search_videos(
select_columns = 'fg.id, COUNT(*) OVER() AS total'
if tag_names:
where_, params_, join_ = tag_names_to_clauses(tag_names)
wheres.append(where_)
# Filter all FileGroups by those that have been tagged with the provided tag names.
tags_stmt, params_ = tag_names_to_sub_select(tag_names)
params.update(params_)
joins.append(join_)
wheres.append(f'fg.id = ANY({tags_stmt})')
if search_str and headline:
headline = ''',

View file

@ -648,10 +648,10 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
wheres.append('model = %(model)s')
if tags:
where_, params_, join_ = tag_names_to_clauses(tags)
wheres.append(where_)
# Filter all FileGroups by those that have been tagged with the provided tag names.
tags_stmt, params_ = tag_names_to_sub_select(tags)
params.update(params_)
joins.append(join_)
wheres.append(f'fg.id = ANY({tags_stmt})')
if search_str and headline:
headline = ''',
@ -681,24 +681,22 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
return results, total
def tag_names_to_clauses(tags: List[str]):
"""Create the SQL necessary to filter the `file_group` table by the provided Tag names."""
params = dict()
def tag_names_to_sub_select(names: List[str]) -> Tuple[str, dict]:
"""Create the SQL necessary to filter FileGroup by the provided Tag names."""
if not names:
return '', dict()
if not tags:
return '', params, ''
where_tags = []
joins = []
for idx, tag_name in enumerate(tags):
where_tags.append(f't{idx}.name = %(tag_name{idx})s')
params[f'tag_name{idx}'] = tag_name
joins.append(f'LEFT JOIN tag_file tf{idx} ON tf{idx}.file_group_id = fg.id '
f'LEFT JOIN tag t{idx} ON t{idx}.id = tf{idx}.tag_id')
where_tags = ' AND '.join(where_tags)
wheres = f'({where_tags})'
join = '\n'.join(joins)
return wheres, params, join
stmt = '''
SELECT
tf.file_group_id
FROM
tag_file tf
LEFT JOIN tag t on t.id = tf.tag_id
GROUP BY file_group_id
-- Match only FileGroups that have at least all the Tag names.
HAVING array_agg(t.name) @> %(tag_names)s::VARCHAR[]
'''
return stmt, dict(tag_names=names)
def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List[dict], int]:

View file

@ -1,6 +1,5 @@
import asyncio
import json
import pathlib
import shutil
import zipfile
from datetime import datetime
@ -11,7 +10,6 @@ from uuid import uuid4
import mock
import pytest
import sqlalchemy.exc
from PIL import Image
from wrolpi.common import timer
@ -543,36 +541,6 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e
assert lib.get_primary_file([example_pdf, example_mobi, example_epub]) == example_epub
@pytest.mark.parametrize(
'tags,wheres,params,join', [
(
['one'],
'(t0.name = %(tag_name0)s)',
dict(tag_name0='one'),
'LEFT JOIN tag_file tf0 ON tf0.file_group_id = fg.id LEFT JOIN tag t0 ON t0.id = tf0.tag_id',
),
(
['one', 'two'],
'(t0.name = %(tag_name0)s AND t1.name = %(tag_name1)s)',
dict(tag_name0='one', tag_name1='two'),
'LEFT JOIN tag_file tf0 ON tf0.file_group_id = fg.id LEFT JOIN tag t0 ON t0.id = tf0.tag_id'
'\nLEFT JOIN tag_file tf1 ON tf1.file_group_id = fg.id LEFT JOIN tag t1 ON t1.id = tf1.tag_id',
),
(
[],
'',
dict(),
'',
),
]
)
def test_tag_names_to_clauses(tags, wheres, params, join):
result = lib.tag_names_to_clauses(tags)
assert result[0] == wheres
assert result[1] == params
assert result[2] == join
def test_get_refresh_progress(test_client, test_session):
request, response = test_client.get('/api/files/refresh_progress')
assert response.status_code == HTTPStatus.OK

View file

@ -5,7 +5,6 @@ import pytest
import yaml
from wrolpi import tags
from wrolpi.dates import now
from wrolpi.errors import FileGroupIsTagged
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup