From 2c8f765f2b60ee17729248d69c545b2adb34ba51 Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 2 Feb 2026 15:15:28 -0700 Subject: [PATCH] Adding statistics page for Archives. --- app/src/api.js | 16 ++++++++ app/src/components/Archive.js | 56 ++++++++++++++++++++++++++ app/src/hooks/customHooks.js | 24 +++++++++++ modules/archive/api.py | 7 ++++ modules/archive/lib.py | 68 +++++++++++++++++++++++++++++++- modules/archive/schema.py | 22 +++++++++++ modules/archive/test/test_lib.py | 33 ++++++++++++++++ 7 files changed, 224 insertions(+), 2 deletions(-) diff --git a/app/src/api.js b/app/src/api.js index cb49930a..183474ce 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -451,6 +451,22 @@ export async function fetchArchiveBrowsers() { return {browsers: [], available: false}; } +export async function getArchiveStatistics() { + const response = await apiGet(`${ARCHIVES_API}/statistics`); + if (response.status === 200) { + return (await response.json())['statistics']; + } else { + const message = await getErrorMessage(response, 'Could not get archive statistics.'); + toast({ + type: 'error', + title: 'Getting Archive Statistics Failed', + description: message, + time: 5000, + }); + } + return response +} + export async function fetchBrowserProfiles() { const response = await apiGet(`${API_URI}/videos/browser-profiles`); if (response.status === 200) { diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index c6684879..1b0849d2 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -66,6 +66,7 @@ import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-ro import Message from "semantic-ui-react/dist/commonjs/collections/Message"; import { useArchive, + useArchiveStatistics, useDockerized, useDomain, useDomains, @@ -1247,6 +1248,59 @@ export function ArchiveRowCells({file}) { } +function ArchiveStatistics() { + useTitle('Archive Statistics'); + + const {statistics} = useArchiveStatistics(); + + if (statistics === null) { + // Request is pending. + return + } else if (statistics === undefined) { + return Unable to fetch Archive Statistics + } + + const {archives, historical, domains} = statistics; + + const archiveNames = [ + {key: 'archives', label: 'Archives'}, + {key: 'sum_size', label: 'Total Size'}, + {key: 'max_size', label: 'Largest Archive'}, + {key: 'week', label: 'Downloads Past Week'}, + {key: 'month', label: 'Downloads Past Month'}, + {key: 'year', label: 'Downloads Past Year'}, + ]; + const historicalNames = [ + {key: 'average_count', label: 'Average Monthly Downloads'}, + {key: 'average_size', label: 'Average Monthly Usage'}, + ]; + const domainNames = [ + {key: 'domains', label: 'Domains'}, + {key: 'tagged_domains', label: 'Tagged Domains'}, + ]; + + const buildSegment = (title, names, stats) => { + return +
{title}
+ + {names.map( + ({key, label}) => + + {stats[key]} + {label} + + )} + + + } + + return <> + {buildSegment('Archives', archiveNames, archives)} + {buildSegment('Historical Archives', historicalNames, historical)} + {buildSegment('Domains', domainNames, domains)} + +} + export function ArchiveRoute() { const location = useLocation(); const path = location.pathname; @@ -1264,6 +1318,7 @@ export function ArchiveRoute() { isActive: () => path.startsWith('/archive/domain') }, {text: 'Settings', to: '/archive/settings'}, + {text: 'Statistics', to: '/archive/statistics'}, ]; return @@ -1272,6 +1327,7 @@ export function ArchiveRoute() { }/> }/> }/> + }/> }/> diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index c0a132af..0601c4d4 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -7,6 +7,7 @@ import { fetchDomains, filesSearch, getArchive, + getArchiveStatistics, getChannel, getConfigs, getDomain, @@ -1151,6 +1152,29 @@ export const useVideoStatistics = () => { return {statistics, fetchStatistics} } +export const useArchiveStatistics = () => { + const [statistics, setStatistics] = useState(null); + + const fetchStatistics = async () => { + try { + let stats = await getArchiveStatistics(); + stats.archives.sum_size = humanFileSize(stats.archives.sum_size); + stats.archives.max_size = humanFileSize(stats.archives.max_size); + stats.historical.average_size = humanFileSize(stats.historical.average_size); + setStatistics(stats); + } catch (e) { + console.error(e); + setStatistics(undefined); // Display error. + } + } + + useEffect(() => { + fetchStatistics(); + }, []); + + return {statistics, fetchStatistics} +} + export const useInventory = (inventoryId) => { const [byCategory, setByCategory] = useState(); const [bySubcategory, setBySubcategory] = useState(); diff --git a/modules/archive/api.py b/modules/archive/api.py index fc161ace..eb89dff3 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -24,6 +24,13 @@ archive_bp = Blueprint('Archive', '/api/archive') logger = logger.getChild(__name__) +@archive_bp.get('/statistics') +@openapi.response(HTTPStatus.OK, schema.ArchiveStatisticsResponse) +async def statistics(_: Request): + ret = await lib.get_statistics() + return json_response(ret, HTTPStatus.OK) + + @archive_bp.get('/') @openapi.description('Get an archive') @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 243d664c..c7ae3e25 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -367,7 +367,8 @@ def format_archive_filename( except KeyError as e: logger.error(f'Invalid variable in archive file_name_format: {e}') # Fallback to default format - return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html') + return trim_file_name( + f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html') def preview_archive_filename(file_name_format: str) -> str: @@ -465,7 +466,8 @@ def format_archive_filename_from_archive( except KeyError as e: logger.error(f'Invalid variable in archive file_name_format: {e}') # Fallback to default format - return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html') + return trim_file_name( + f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html') @dataclass @@ -1420,6 +1422,68 @@ async def singlefile_to_archive(singlefile: bytes, destination: pathlib.Path = N return archive +async def get_statistics(): + """Get statistics about Archives and Domain Collections.""" + with get_db_curs() as curs: + curs.execute(''' + SELECT + -- total archives + COUNT(a.id) AS "archives", + -- total archives downloaded over the past week/month/year + COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 week') AS "week", + COUNT(a.id) + FILTER (WHERE fg.download_datetime >= current_date - interval '1 month') AS "month", + COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 year') AS "year", + -- sum of all archive file sizes + COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size", + -- largest archive + COALESCE(MAX(fg.size), 0) AS "max_size" + FROM archive a + LEFT JOIN file_group fg on a.file_group_id = fg.id + ''') + archive_stats = dict(curs.fetchone()) + + # Get the total archives downloaded every month for the past two years. + curs.execute(''' + SELECT DATE_TRUNC('month', months.a), + COUNT(archive.id)::BIGINT, + SUM(size)::BIGINT AS "size" + FROM generate_series( + date_trunc('month', current_date) - interval '2 years', + date_trunc('month', current_date) - interval '1 month', + '1 month'::interval) AS months(a), + archive + LEFT JOIN file_group fg on archive.file_group_id = fg.id + WHERE fg.download_datetime >= date_trunc('month', months.a) + AND fg.download_datetime < date_trunc('month', months.a) + interval '1 month' + AND fg.download_datetime IS NOT NULL + GROUP BY 1 + ORDER BY 1 + ''') + monthly_archives = [dict(i) for i in curs.fetchall()] + + historical_stats = dict(monthly_archives=monthly_archives) + historical_stats['average_count'] = (sum(i['count'] for i in monthly_archives) // len(monthly_archives)) \ + if monthly_archives else 0 + historical_stats['average_size'] = (sum(i['size'] for i in monthly_archives) // len(monthly_archives)) \ + if monthly_archives else 0 + + curs.execute(''' + SELECT COUNT(c.id) AS "domains", + COUNT(c.id) FILTER ( WHERE col.tag_id IS NOT NULL ) AS "tagged_domains" + FROM collection c + LEFT JOIN collection col on col.id = c.id + WHERE c.kind = 'domain' + ''') + domain_stats = dict(curs.fetchone()) + ret = dict(statistics=dict( + archives=archive_stats, + domains=domain_stats, + historical=historical_stats, + )) + return ret + + async def generate_archive_screenshot(archive_id: int) -> pathlib.Path: """ Generate a screenshot for an existing Archive that doesn't have one. diff --git a/modules/archive/schema.py b/modules/archive/schema.py index dde5853d..7e9b5cd0 100644 --- a/modules/archive/schema.py +++ b/modules/archive/schema.py @@ -35,3 +35,25 @@ class ArchiveSearchResponse: @dataclass class ArchiveFileFormatRequest: archive_file_format: str + + +@dataclass +class ArchiveStatistics: + archives: int + week: int + month: int + year: int + sum_size: int + max_size: int + + +@dataclass +class DomainStatistics: + domains: int + tagged_domains: int + + +@dataclass +class ArchiveStatisticsResponse: + archives: ArchiveStatistics + domains: DomainStatistics diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 1e6446ce..5502dc37 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -1357,3 +1357,36 @@ def test_format_archive_filename_escapes_special_chars(test_directory, fake_now) assert '/' not in result.split('/')[-1] # No slashes in filename part assert '2000-01-01' in result assert result.endswith('.html') + + +@pytest.mark.asyncio +async def test_get_statistics(test_session, archive_factory): + """get_statistics returns archive and domain statistics.""" + from modules.archive.lib import get_statistics + + # Can get statistics in empty DB. + result = await get_statistics() + assert 'statistics' in result + assert 'archives' in result['statistics'] + assert 'domains' in result['statistics'] + assert 'historical' in result['statistics'] + + # Empty stats should have zero values. + stats = result['statistics'] + assert stats['archives']['archives'] == 0 + assert stats['domains']['domains'] == 0 + + # Create some archives. + archive1 = archive_factory('example.com') + archive2 = archive_factory('example.com') + archive3 = archive_factory('example.org') + archive4 = archive_factory() # No domain + test_session.commit() + + result = await get_statistics() + stats = result['statistics'] + + # Should count all 4 archives. + assert stats['archives']['archives'] == 4 + # Should count 2 domain collections (example.com and example.org). + assert stats['domains']['domains'] == 2