diff --git a/app/src/api.js b/app/src/api.js
index cb49930a..183474ce 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -451,6 +451,22 @@ export async function fetchArchiveBrowsers() {
return {browsers: [], available: false};
}
+export async function getArchiveStatistics() {
+ const response = await apiGet(`${ARCHIVES_API}/statistics`);
+ if (response.status === 200) {
+ return (await response.json())['statistics'];
+ } else {
+ const message = await getErrorMessage(response, 'Could not get archive statistics.');
+ toast({
+ type: 'error',
+ title: 'Getting Archive Statistics Failed',
+ description: message,
+ time: 5000,
+ });
+ }
+ return response
+}
+
export async function fetchBrowserProfiles() {
const response = await apiGet(`${API_URI}/videos/browser-profiles`);
if (response.status === 200) {
diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js
index c6684879..1b0849d2 100644
--- a/app/src/components/Archive.js
+++ b/app/src/components/Archive.js
@@ -66,6 +66,7 @@ import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-ro
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {
useArchive,
+ useArchiveStatistics,
useDockerized,
useDomain,
useDomains,
@@ -1247,6 +1248,59 @@ export function ArchiveRowCells({file}) {
}
+function ArchiveStatistics() {
+ useTitle('Archive Statistics');
+
+ const {statistics} = useArchiveStatistics();
+
+ if (statistics === null) {
+ // Request is pending.
+ return
+ } else if (statistics === undefined) {
+ return Unable to fetch Archive Statistics
+ }
+
+ const {archives, historical, domains} = statistics;
+
+ const archiveNames = [
+ {key: 'archives', label: 'Archives'},
+ {key: 'sum_size', label: 'Total Size'},
+ {key: 'max_size', label: 'Largest Archive'},
+ {key: 'week', label: 'Downloads Past Week'},
+ {key: 'month', label: 'Downloads Past Month'},
+ {key: 'year', label: 'Downloads Past Year'},
+ ];
+ const historicalNames = [
+ {key: 'average_count', label: 'Average Monthly Downloads'},
+ {key: 'average_size', label: 'Average Monthly Usage'},
+ ];
+ const domainNames = [
+ {key: 'domains', label: 'Domains'},
+ {key: 'tagged_domains', label: 'Tagged Domains'},
+ ];
+
+ const buildSegment = (title, names, stats) => {
+ return
+
+
+ {names.map(
+ ({key, label}) =>
+
+ {stats[key]}
+ {label}
+
+ )}
+
+
+ }
+
+ return <>
+ {buildSegment('Archives', archiveNames, archives)}
+ {buildSegment('Historical Archives', historicalNames, historical)}
+ {buildSegment('Domains', domainNames, domains)}
+
+}
+
export function ArchiveRoute() {
const location = useLocation();
const path = location.pathname;
@@ -1264,6 +1318,7 @@ export function ArchiveRoute() {
isActive: () => path.startsWith('/archive/domain')
},
{text: 'Settings', to: '/archive/settings'},
+ {text: 'Statistics', to: '/archive/statistics'},
];
return
@@ -1272,6 +1327,7 @@ export function ArchiveRoute() {
}/>
}/>
}/>
+ }/>
}/>
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js
index c0a132af..0601c4d4 100644
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@@ -7,6 +7,7 @@ import {
fetchDomains,
filesSearch,
getArchive,
+ getArchiveStatistics,
getChannel,
getConfigs,
getDomain,
@@ -1151,6 +1152,29 @@ export const useVideoStatistics = () => {
return {statistics, fetchStatistics}
}
+export const useArchiveStatistics = () => {
+ const [statistics, setStatistics] = useState(null);
+
+ const fetchStatistics = async () => {
+ try {
+ let stats = await getArchiveStatistics();
+ stats.archives.sum_size = humanFileSize(stats.archives.sum_size);
+ stats.archives.max_size = humanFileSize(stats.archives.max_size);
+ stats.historical.average_size = humanFileSize(stats.historical.average_size);
+ setStatistics(stats);
+ } catch (e) {
+ console.error(e);
+ setStatistics(undefined); // Display error.
+ }
+ }
+
+ useEffect(() => {
+ fetchStatistics();
+ }, []);
+
+ return {statistics, fetchStatistics}
+}
+
export const useInventory = (inventoryId) => {
const [byCategory, setByCategory] = useState();
const [bySubcategory, setBySubcategory] = useState();
diff --git a/modules/archive/api.py b/modules/archive/api.py
index fc161ace..eb89dff3 100644
--- a/modules/archive/api.py
+++ b/modules/archive/api.py
@@ -24,6 +24,13 @@ archive_bp = Blueprint('Archive', '/api/archive')
logger = logger.getChild(__name__)
+@archive_bp.get('/statistics')
+@openapi.response(HTTPStatus.OK, schema.ArchiveStatisticsResponse)
+async def statistics(_: Request):
+ ret = await lib.get_statistics()
+ return json_response(ret, HTTPStatus.OK)
+
+
@archive_bp.get('/')
@openapi.description('Get an archive')
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 243d664c..c7ae3e25 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -367,7 +367,8 @@ def format_archive_filename(
except KeyError as e:
logger.error(f'Invalid variable in archive file_name_format: {e}')
# Fallback to default format
- return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
+ return trim_file_name(
+ f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
def preview_archive_filename(file_name_format: str) -> str:
@@ -465,7 +466,8 @@ def format_archive_filename_from_archive(
except KeyError as e:
logger.error(f'Invalid variable in archive file_name_format: {e}')
# Fallback to default format
- return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
+ return trim_file_name(
+ f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
@dataclass
@@ -1420,6 +1422,68 @@ async def singlefile_to_archive(singlefile: bytes, destination: pathlib.Path = N
return archive
+async def get_statistics():
+ """Get statistics about Archives and Domain Collections."""
+ with get_db_curs() as curs:
+ curs.execute('''
+ SELECT
+ -- total archives
+ COUNT(a.id) AS "archives",
+ -- total archives downloaded over the past week/month/year
+ COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 week') AS "week",
+ COUNT(a.id)
+ FILTER (WHERE fg.download_datetime >= current_date - interval '1 month') AS "month",
+ COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 year') AS "year",
+ -- sum of all archive file sizes
+ COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size",
+ -- largest archive
+ COALESCE(MAX(fg.size), 0) AS "max_size"
+ FROM archive a
+ LEFT JOIN file_group fg on a.file_group_id = fg.id
+ ''')
+ archive_stats = dict(curs.fetchone())
+
+ # Get the total archives downloaded every month for the past two years.
+ curs.execute('''
+ SELECT DATE_TRUNC('month', months.a),
+ COUNT(archive.id)::BIGINT,
+ SUM(size)::BIGINT AS "size"
+ FROM generate_series(
+ date_trunc('month', current_date) - interval '2 years',
+ date_trunc('month', current_date) - interval '1 month',
+ '1 month'::interval) AS months(a),
+ archive
+ LEFT JOIN file_group fg on archive.file_group_id = fg.id
+ WHERE fg.download_datetime >= date_trunc('month', months.a)
+ AND fg.download_datetime < date_trunc('month', months.a) + interval '1 month'
+ AND fg.download_datetime IS NOT NULL
+ GROUP BY 1
+ ORDER BY 1
+ ''')
+ monthly_archives = [dict(i) for i in curs.fetchall()]
+
+ historical_stats = dict(monthly_archives=monthly_archives)
+ historical_stats['average_count'] = (sum(i['count'] for i in monthly_archives) // len(monthly_archives)) \
+ if monthly_archives else 0
+ historical_stats['average_size'] = (sum(i['size'] for i in monthly_archives) // len(monthly_archives)) \
+ if monthly_archives else 0
+
+ curs.execute('''
+ SELECT COUNT(c.id) AS "domains",
+ COUNT(c.id) FILTER ( WHERE col.tag_id IS NOT NULL ) AS "tagged_domains"
+ FROM collection c
+ LEFT JOIN collection col on col.id = c.id
+ WHERE c.kind = 'domain'
+ ''')
+ domain_stats = dict(curs.fetchone())
+ ret = dict(statistics=dict(
+ archives=archive_stats,
+ domains=domain_stats,
+ historical=historical_stats,
+ ))
+ return ret
+
+
async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
"""
Generate a screenshot for an existing Archive that doesn't have one.
diff --git a/modules/archive/schema.py b/modules/archive/schema.py
index dde5853d..7e9b5cd0 100644
--- a/modules/archive/schema.py
+++ b/modules/archive/schema.py
@@ -35,3 +35,25 @@ class ArchiveSearchResponse:
@dataclass
class ArchiveFileFormatRequest:
archive_file_format: str
+
+
+@dataclass
+class ArchiveStatistics:
+ archives: int
+ week: int
+ month: int
+ year: int
+ sum_size: int
+ max_size: int
+
+
+@dataclass
+class DomainStatistics:
+ domains: int
+ tagged_domains: int
+
+
+@dataclass
+class ArchiveStatisticsResponse:
+ archives: ArchiveStatistics
+ domains: DomainStatistics
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index 1e6446ce..5502dc37 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -1357,3 +1357,36 @@ def test_format_archive_filename_escapes_special_chars(test_directory, fake_now)
assert '/' not in result.split('/')[-1] # No slashes in filename part
assert '2000-01-01' in result
assert result.endswith('.html')
+
+
+@pytest.mark.asyncio
+async def test_get_statistics(test_session, archive_factory):
+ """get_statistics returns archive and domain statistics."""
+ from modules.archive.lib import get_statistics
+
+ # Can get statistics in empty DB.
+ result = await get_statistics()
+ assert 'statistics' in result
+ assert 'archives' in result['statistics']
+ assert 'domains' in result['statistics']
+ assert 'historical' in result['statistics']
+
+ # Empty stats should have zero values.
+ stats = result['statistics']
+ assert stats['archives']['archives'] == 0
+ assert stats['domains']['domains'] == 0
+
+ # Create some archives.
+ archive1 = archive_factory('example.com')
+ archive2 = archive_factory('example.com')
+ archive3 = archive_factory('example.org')
+ archive4 = archive_factory() # No domain
+ test_session.commit()
+
+ result = await get_statistics()
+ stats = result['statistics']
+
+ # Should count all 4 archives.
+ assert stats['archives']['archives'] == 4
+ # Should count 2 domain collections (example.com and example.org).
+ assert stats['domains']['domains'] == 2