From 9da3b3d8b260d0dcfc6495ecd15e2fa67ca41aaa Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 4 Jan 2022 12:15:11 -0700 Subject: [PATCH] Creating api_param_limiter which can be used to restrict limit or offset. --- modules/archive/api.py | 10 +++++++--- modules/videos/common.py | 11 ++--------- modules/videos/video/api.py | 4 ++-- wrolpi/common.py | 23 +++++++++++++++++++++++ wrolpi/test/test_common.py | 24 +++++++++++++++++++++++- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/modules/archive/api.py b/modules/archive/api.py index 22643cdc..dc2027e0 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -4,7 +4,7 @@ from http import HTTPStatus from requests import Request from sanic import response -from wrolpi.common import logger, wrol_mode_check +from wrolpi.common import logger, wrol_mode_check, api_param_limiter from wrolpi.errors import ValidationError from wrolpi.root_api import get_blueprint, json_response from wrolpi.schema import validate_doc, JSONErrorResponse @@ -44,6 +44,10 @@ async def fetch_domains(_: Request): return json_response({'domains': domains, 'totals': {'domains': len(domains)}}) +archive_limit_limiter = api_param_limiter(100) +archive_offset_limiter = api_param_limiter(100, 0) + + @bp.post('/search') @validate_doc( 'Search archive contents and titles', @@ -55,8 +59,8 @@ async def search_archives(_: Request, data: dict): try: search_str = data.get('search_str') domain = data.get('domain') - limit = int(data.get('limit', 20)) - offset = int(data.get('offset', 0)) + limit = archive_limit_limiter(data.get('limit')) + offset = archive_offset_limiter(data.get('offset')) except Exception as e: raise ValidationError('Unable to validate search queries') from e diff --git a/modules/videos/common.py b/modules/videos/common.py index 090b6a0a..645d8f08 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -13,7 +13,7 @@ from sqlalchemy.orm import Session from wrolpi import before_startup from wrolpi.common import logger, get_config, iterify, chunks, get_media_directory, \ - minimize_dict + minimize_dict, api_param_limiter from wrolpi.db import get_db_session, get_db_curs from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \ ChannelLinkConflict, ChannelDirectoryConflict, ChannelSourceIdConflict @@ -47,14 +47,7 @@ def load_channels_config() -> dict: return channels -def get_allowed_limit(limit: int) -> int: - """ - Return the video limit int if it was passed, unless it is over the maximum limit. If no limit is provided, return - the default limit. - """ - if limit: - return min(int(limit), VIDEO_QUERY_MAX_LIMIT) - return VIDEO_QUERY_LIMIT +video_limiter = api_param_limiter(100) class ConfigError(Exception): diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index 3f63e6be..347fb212 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -10,7 +10,7 @@ from wrolpi.errors import ValidationError, InvalidOrderBy from wrolpi.root_api import json_response from wrolpi.schema import validate_doc, JSONErrorResponse from .lib import get_video, VIDEO_ORDERS, DEFAULT_VIDEO_ORDER, video_search, get_video_for_app -from ..common import get_matching_directories, get_allowed_limit +from ..common import get_matching_directories, video_limiter from ..lib import save_channels_config from ..schema import VideoResponse, VideoSearchRequest, VideoSearchResponse, \ DirectoriesResponse, DirectoriesRequest @@ -45,7 +45,7 @@ async def search(_: Request, data: dict): channel_link = data.get('channel_link') order_by = data.get('order_by', DEFAULT_VIDEO_ORDER) offset = int(data.get('offset', 0)) - limit = get_allowed_limit(data.get('limit')) + limit = video_limiter(data.get('limit')) filters = data.get('filters', None) except Exception as e: raise ValidationError('Unable to validate search queries') from e diff --git a/wrolpi/common.py b/wrolpi/common.py index 82f16e0b..08886e69 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -481,3 +481,26 @@ def import_modules(): logger.fatal('No modules could be found!', exc_info=e) raise return modules + + +def api_param_limiter(maximum: int, default: int = 20) -> callable: + """ + Create a function which restricts the maximum number that can be returned. + Useful for restricting API limit params. + + >>> limiter = api_param_limiter(100) + >>> limiter(0) + 0 + >>> limiter(100) + 100 + >>> limiter(150) + 150 + """ + + def limiter_(i: int) -> int: + if not i: + # Limit is not valid, use the default. + return default + return min(int(i), maximum) + + return limiter_ diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 579bf08e..a00a5764 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -4,7 +4,7 @@ from datetime import date, datetime import pytest from sanic_openapi import doc -from wrolpi.common import combine_dicts, insert_parameter, date_range +from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter from wrolpi.dates import set_timezone, now from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone from wrolpi.schema import validate_data @@ -293,3 +293,25 @@ class TestCommon(unittest.TestCase): finally: # Restore the timezone before the test. set_timezone(original_timezone) + + +@pytest.mark.parametrize( + 'i,expected', [ + (1, 1), + (100, 100), + (150, 100), + (-1, -1), + (-1.0, -1.0), + (1.0, 1.0), + (100.0, 100.0), + (150.0, 100.0), + ('1', 1), + (None, 20), + (0, 20), + (0.0, 20), + ('', 20), + ] +) +def test_api_param_limiter(i, expected): + limiter = api_param_limiter(100) # should never return an integer greater than 100. + assert limiter(i) == expected