From 1d69e5417a96bf0fda1c49fd6f65a5147438ac40 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 7 Jan 2022 10:33:50 -0700 Subject: [PATCH] Making calling optional_session optional. --- modules/videos/lib.py | 9 +++------ wrolpi/common.py | 14 ++++++++++++- wrolpi/db.py | 46 ++++++++++++++++++++++++++++++------------ wrolpi/downloader.py | 18 ++++++----------- wrolpi/test/test_db.py | 25 +++++++++++++++++++---- 5 files changed, 76 insertions(+), 36 deletions(-) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index ef81e4e4..690a54d9 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -11,7 +11,7 @@ from sqlalchemy.orm import Session from wrolpi.common import logger, chunks, get_config from wrolpi.common import save_settings_config -from wrolpi.db import get_db_curs, get_db_session +from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.vars import PYTEST from .captions import insert_bulk_captions, process_captions from .common import generate_bulk_posters, get_bulk_video_info_json, get_bulk_video_size, \ @@ -280,15 +280,12 @@ def get_channels_config(session: Session) -> dict: return dict(channels=channels, favorites=favorites) +@optional_session() def save_channels_config(session=None, preserve_favorites: bool = True): """ Pull the Channel information from the DB, save it to the config. """ - if session: - config = get_channels_config(session) - else: - with get_db_session() as session: - config = get_channels_config(session) + config = get_channels_config(session) if preserve_favorites: config['favorites'].update(get_config().get('favorites', {})) save_settings_config(config) diff --git a/wrolpi/common.py b/wrolpi/common.py index a228f5a8..c4f005d6 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -11,7 +11,7 @@ import string import tempfile from datetime import datetime, date from functools import wraps -from itertools import islice +from itertools import islice, filterfalse, tee from multiprocessing import Event, Queue from pathlib import Path from typing import Union, Callable, Tuple, Dict, Mapping, List, Iterable, Optional @@ -514,3 +514,15 @@ def api_param_limiter(maximum: int, default: int = 20) -> callable: def temporary_directory_path(*a, **kw): with tempfile.TemporaryDirectory(*a, **kw) as d: yield pathlib.Path(d) + + +def partition(pred, iterable): + """ + Use a predicate to partition entries into false entries + and true entries + >>> is_even = lambda i: not i % 2 + >>> partition(is_even, range(10)) + ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) + """ + t1, t2 = tee(iterable) + return list(filter(pred, t2)), list(filterfalse(pred, t1)) diff --git a/wrolpi/db.py b/wrolpi/db.py index 428bcda6..db5539ef 100644 --- a/wrolpi/db.py +++ b/wrolpi/db.py @@ -1,6 +1,6 @@ from contextlib import contextmanager from functools import wraps -from typing import ContextManager, Tuple, List +from typing import ContextManager, Tuple, List, Union import psycopg2 import sqlalchemy.exc @@ -8,7 +8,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, Session from sqlalchemy.pool import NullPool -from wrolpi.common import logger, Base +from wrolpi.common import logger, Base, partition from wrolpi.vars import DB_NAME, DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DOCKERIZED, PYTEST logger = logger.getChild(__name__) @@ -114,27 +114,47 @@ def get_db_curs(commit: bool = False): connection.rollback() -def optional_session(commit: bool = False): +def optional_session(commit: Union[callable, bool] = False): """ - Wraps a function, if a Session is passed as keyword `session`, it will be used. Otherwise, a new session will be + Wraps a function, if a Session is passed it will be used. Otherwise, a new session will be created and passed to the function. """ - def wrapper(func: callable): - @wraps(func) - def wrapped(*a, session: Session = None, **kw): + def find_session(*f_args, session: Session = None, **f_kwargs): + session = f_kwargs.get('session', session) + if not session: + # Find the session in the args. + session, new_args = partition(lambda i: isinstance(i, Session), f_args) if session: - return func(*a, **kw, session=session) - else: - with get_db_session(commit=commit) as session: - return func(*a, **kw, session=session) + session = session[0] + f_args = tuple(new_args) + return session, f_args, f_kwargs - return wrapped + def call_func(func, session, args, kwargs): + if session: + return func(*args, session=session, **kwargs) + else: + with get_db_session() as session: + return func(*args, session=session, **kwargs) + + def wrapper(*w_args, **w_kwargs): + if len(w_args) == 1 and len(w_kwargs) == 0 and callable(w_args[0]): + func = w_args[0] + + @wraps(func) + def wrapped(*args, session: Session = None, **kwargs): + session, args, kwargs = find_session(*args, session=session, **kwargs) + return call_func(func, session, args, kwargs) + + return wrapped + else: + session_, w_args, w_kwargs = find_session(*w_args, **w_kwargs) + return call_func(commit, session_, w_args, w_kwargs) return wrapper -@optional_session() +@optional_session def get_ranked_models(ranked_ids: List[int], model: Base, session: Session = None) -> List[Base]: """ Get all objects whose ids are in the `ranked_ids`, order them by their position in `ranked_ids`. diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index fa2c4b94..4e335c9a 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -15,7 +15,7 @@ from sqlalchemy.orm import Session from wrolpi.common import Base, ModelHelper, logger, iterify from wrolpi.dates import TZDateTime, now -from wrolpi.db import get_db_session, get_db_curs, get_db_context +from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError from wrolpi.vars import PYTEST @@ -309,14 +309,12 @@ class DownloadManager: return downloads + @optional_session def _do_downloads(self, session=None): """ This method calls the Downloader's do_download method. """ # This is a long-running function, lets get a session that can be used for a long time. - if not session: - _, session = get_db_context() - try: downloads = self.get_new_downloads(session) @@ -422,12 +420,11 @@ class DownloadManager: for sort in cls.DOWNLOAD_SORT: yield from grouped_by_statuses[sort] # converted to a list by iterify() + @optional_session def get_recurring_downloads(self, session: Session = None, limit: int = None): """ Get all Downloads that will be downloaded in the future. """ - if not session: - _, session = get_db_context() query = session.query(Download).filter( Download.frequency != None # noqa ).order_by( @@ -440,12 +437,11 @@ class DownloadManager: downloads = self._downloads_sorter(downloads) return downloads + @optional_session def get_once_downloads(self, session: Session = None, limit: int = None): """ Get all Downloads that will not reoccur. """ - if not session: - _, session = get_db_context() query = session.query(Download).filter( Download.frequency == None # noqa ).order_by( @@ -458,13 +454,12 @@ class DownloadManager: downloads = self._downloads_sorter(downloads) return downloads + @optional_session def renew_recurring_downloads(self, session: Session = None): """ Mark any recurring downloads that are due for download as "new". Start a download. """ now_ = now() - if not session: - _, session = get_db_context() recurring = self.get_recurring_downloads(session) renewed = False @@ -493,12 +488,11 @@ class DownloadManager: elif id: return query.filter_by(id=id_).one_or_none() + @optional_session def delete_download(self, download_id: int, session: Session = None): """ Delete a Download. Returns True if a Download was deleted, otherwise return False. """ - if not session: - _, session = get_db_context() download = self.get_download(session, id_=download_id) if download: session.delete(download) diff --git a/wrolpi/test/test_db.py b/wrolpi/test/test_db.py index f44624e4..af477c9a 100644 --- a/wrolpi/test/test_db.py +++ b/wrolpi/test/test_db.py @@ -1,10 +1,12 @@ -import pytest from sqlalchemy.orm import Session from wrolpi.db import optional_session def test_optional_session(test_session): + """ + `optional_session` wraps a function and provides a session if one was not passed. + """ url = test_session.get_bind().url @optional_session() @@ -13,7 +15,22 @@ def test_optional_session(test_session): func() func(session=test_session) + func(test_session) - with pytest.raises(TypeError): - # keyword must be used. - func(test_session) + # Commit can be passed. + @optional_session(commit=True) + def func(session: Session): + assert session.get_bind().url == url + + func() + func(session=test_session) + func(test_session) + + # Wrapper is not called. + @optional_session + def func(session: Session): + assert session.get_bind().url == url + + func() + func(session=test_session) + func(test_session)