mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 23:46:18 -05:00
Adding statistics page for Archives.
This commit is contained in:
parent
ad59d67171
commit
2c8f765f2b
7 changed files with 224 additions and 2 deletions
|
|
@ -24,6 +24,13 @@ archive_bp = Blueprint('Archive', '/api/archive')
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
@archive_bp.get('/statistics')
|
||||
@openapi.response(HTTPStatus.OK, schema.ArchiveStatisticsResponse)
|
||||
async def statistics(_: Request):
|
||||
ret = await lib.get_statistics()
|
||||
return json_response(ret, HTTPStatus.OK)
|
||||
|
||||
|
||||
@archive_bp.get('/<archive_id:int>')
|
||||
@openapi.description('Get an archive')
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
|
|
|
|||
|
|
@ -367,7 +367,8 @@ def format_archive_filename(
|
|||
except KeyError as e:
|
||||
logger.error(f'Invalid variable in archive file_name_format: {e}')
|
||||
# Fallback to default format
|
||||
return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
|
||||
return trim_file_name(
|
||||
f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
|
||||
|
||||
|
||||
def preview_archive_filename(file_name_format: str) -> str:
|
||||
|
|
@ -465,7 +466,8 @@ def format_archive_filename_from_archive(
|
|||
except KeyError as e:
|
||||
logger.error(f'Invalid variable in archive file_name_format: {e}')
|
||||
# Fallback to default format
|
||||
return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
|
||||
return trim_file_name(
|
||||
f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -1420,6 +1422,68 @@ async def singlefile_to_archive(singlefile: bytes, destination: pathlib.Path = N
|
|||
return archive
|
||||
|
||||
|
||||
async def get_statistics():
|
||||
"""Get statistics about Archives and Domain Collections."""
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('''
|
||||
SELECT
|
||||
-- total archives
|
||||
COUNT(a.id) AS "archives",
|
||||
-- total archives downloaded over the past week/month/year
|
||||
COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 week') AS "week",
|
||||
COUNT(a.id)
|
||||
FILTER (WHERE fg.download_datetime >= current_date - interval '1 month') AS "month",
|
||||
COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 year') AS "year",
|
||||
-- sum of all archive file sizes
|
||||
COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size",
|
||||
-- largest archive
|
||||
COALESCE(MAX(fg.size), 0) AS "max_size"
|
||||
FROM archive a
|
||||
LEFT JOIN file_group fg on a.file_group_id = fg.id
|
||||
''')
|
||||
archive_stats = dict(curs.fetchone())
|
||||
|
||||
# Get the total archives downloaded every month for the past two years.
|
||||
curs.execute('''
|
||||
SELECT DATE_TRUNC('month', months.a),
|
||||
COUNT(archive.id)::BIGINT,
|
||||
SUM(size)::BIGINT AS "size"
|
||||
FROM generate_series(
|
||||
date_trunc('month', current_date) - interval '2 years',
|
||||
date_trunc('month', current_date) - interval '1 month',
|
||||
'1 month'::interval) AS months(a),
|
||||
archive
|
||||
LEFT JOIN file_group fg on archive.file_group_id = fg.id
|
||||
WHERE fg.download_datetime >= date_trunc('month', months.a)
|
||||
AND fg.download_datetime < date_trunc('month', months.a) + interval '1 month'
|
||||
AND fg.download_datetime IS NOT NULL
|
||||
GROUP BY 1
|
||||
ORDER BY 1
|
||||
''')
|
||||
monthly_archives = [dict(i) for i in curs.fetchall()]
|
||||
|
||||
historical_stats = dict(monthly_archives=monthly_archives)
|
||||
historical_stats['average_count'] = (sum(i['count'] for i in monthly_archives) // len(monthly_archives)) \
|
||||
if monthly_archives else 0
|
||||
historical_stats['average_size'] = (sum(i['size'] for i in monthly_archives) // len(monthly_archives)) \
|
||||
if monthly_archives else 0
|
||||
|
||||
curs.execute('''
|
||||
SELECT COUNT(c.id) AS "domains",
|
||||
COUNT(c.id) FILTER ( WHERE col.tag_id IS NOT NULL ) AS "tagged_domains"
|
||||
FROM collection c
|
||||
LEFT JOIN collection col on col.id = c.id
|
||||
WHERE c.kind = 'domain'
|
||||
''')
|
||||
domain_stats = dict(curs.fetchone())
|
||||
ret = dict(statistics=dict(
|
||||
archives=archive_stats,
|
||||
domains=domain_stats,
|
||||
historical=historical_stats,
|
||||
))
|
||||
return ret
|
||||
|
||||
|
||||
async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
|
||||
"""
|
||||
Generate a screenshot for an existing Archive that doesn't have one.
|
||||
|
|
|
|||
|
|
@ -35,3 +35,25 @@ class ArchiveSearchResponse:
|
|||
@dataclass
|
||||
class ArchiveFileFormatRequest:
|
||||
archive_file_format: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArchiveStatistics:
|
||||
archives: int
|
||||
week: int
|
||||
month: int
|
||||
year: int
|
||||
sum_size: int
|
||||
max_size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomainStatistics:
|
||||
domains: int
|
||||
tagged_domains: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArchiveStatisticsResponse:
|
||||
archives: ArchiveStatistics
|
||||
domains: DomainStatistics
|
||||
|
|
|
|||
|
|
@ -1357,3 +1357,36 @@ def test_format_archive_filename_escapes_special_chars(test_directory, fake_now)
|
|||
assert '/' not in result.split('/')[-1] # No slashes in filename part
|
||||
assert '2000-01-01' in result
|
||||
assert result.endswith('.html')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics(test_session, archive_factory):
|
||||
"""get_statistics returns archive and domain statistics."""
|
||||
from modules.archive.lib import get_statistics
|
||||
|
||||
# Can get statistics in empty DB.
|
||||
result = await get_statistics()
|
||||
assert 'statistics' in result
|
||||
assert 'archives' in result['statistics']
|
||||
assert 'domains' in result['statistics']
|
||||
assert 'historical' in result['statistics']
|
||||
|
||||
# Empty stats should have zero values.
|
||||
stats = result['statistics']
|
||||
assert stats['archives']['archives'] == 0
|
||||
assert stats['domains']['domains'] == 0
|
||||
|
||||
# Create some archives.
|
||||
archive1 = archive_factory('example.com')
|
||||
archive2 = archive_factory('example.com')
|
||||
archive3 = archive_factory('example.org')
|
||||
archive4 = archive_factory() # No domain
|
||||
test_session.commit()
|
||||
|
||||
result = await get_statistics()
|
||||
stats = result['statistics']
|
||||
|
||||
# Should count all 4 archives.
|
||||
assert stats['archives']['archives'] == 4
|
||||
# Should count 2 domain collections (example.com and example.org).
|
||||
assert stats['domains']['domains'] == 2
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue