Adding statistics page for Archives.

app/src/api.js

@@ -451,6 +451,22 @@ export async function fetchArchiveBrowsers() {
     return {browsers: [], available: false};
 }
 
+export async function getArchiveStatistics() {
+    const response = await apiGet(`${ARCHIVES_API}/statistics`);
+    if (response.status === 200) {
+        return (await response.json())['statistics'];
+    } else {
+        const message = await getErrorMessage(response, 'Could not get archive statistics.');
+        toast({
+            type: 'error',
+            title: 'Getting Archive Statistics Failed',
+            description: message,
+            time: 5000,
+        });
+    }
+    return response
+}
+
 export async function fetchBrowserProfiles() {
     const response = await apiGet(`${API_URI}/videos/browser-profiles`);
     if (response.status === 200) {

app/src/components/Archive.js

@@ -66,6 +66,7 @@ import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-ro
 import Message from "semantic-ui-react/dist/commonjs/collections/Message";
 import {
     useArchive,
+    useArchiveStatistics,
     useDockerized,
     useDomain,
     useDomains,
@@ -1247,6 +1248,59 @@ export function ArchiveRowCells({file}) {
     </React.Fragment>
 }
 
+function ArchiveStatistics() {
+    useTitle('Archive Statistics');
+
+    const {statistics} = useArchiveStatistics();
+
+    if (statistics === null) {
+        // Request is pending.
+        return <Loader active inline='centered'/>
+    } else if (statistics === undefined) {
+        return <ErrorMessage>Unable to fetch Archive Statistics</ErrorMessage>
+    }
+
+    const {archives, historical, domains} = statistics;
+
+    const archiveNames = [
+        {key: 'archives', label: 'Archives'},
+        {key: 'sum_size', label: 'Total Size'},
+        {key: 'max_size', label: 'Largest Archive'},
+        {key: 'week', label: 'Downloads Past Week'},
+        {key: 'month', label: 'Downloads Past Month'},
+        {key: 'year', label: 'Downloads Past Year'},
+    ];
+    const historicalNames = [
+        {key: 'average_count', label: 'Average Monthly Downloads'},
+        {key: 'average_size', label: 'Average Monthly Usage'},
+    ];
+    const domainNames = [
+        {key: 'domains', label: 'Domains'},
+        {key: 'tagged_domains', label: 'Tagged Domains'},
+    ];
+
+    const buildSegment = (title, names, stats) => {
+        return <Segment>
+            <Header textAlign='center' as='h1'>{title}</Header>
+            <Statistic.Group>
+                {names.map(
+                    ({key, label}) =>
+                        <Statistic key={key} style={{margin: '2em'}}>
+                            <StatisticValue>{stats[key]}</StatisticValue>
+                            <StatisticLabel>{label}</StatisticLabel>
+                        </Statistic>
+                )}
+            </Statistic.Group>
+        </Segment>
+    }
+
+    return <>
+        {buildSegment('Archives', archiveNames, archives)}
+        {buildSegment('Historical Archives', historicalNames, historical)}
+        {buildSegment('Domains', domainNames, domains)}
+    </>
+}
+
 export function ArchiveRoute() {
     const location = useLocation();
     const path = location.pathname;
@@ -1264,6 +1318,7 @@ export function ArchiveRoute() {
             isActive: () => path.startsWith('/archive/domain')
         },
         {text: 'Settings', to: '/archive/settings'},
+        {text: 'Statistics', to: '/archive/statistics'},
     ];
     return <PageContainer>
         <TabLinks links={links}/>
@@ -1272,6 +1327,7 @@ export function ArchiveRoute() {
             <Route path='domains' element={<DomainsPage/>}/>
             <Route path='domain/:domainId/edit' element={<DomainEditPage/>}/>
             <Route path='settings' element={<ArchiveSettingsPage/>}/>
+            <Route path='statistics' element={<ArchiveStatistics/>}/>
             <Route path=':archiveId' element={<ArchivePage/>}/>
         </Routes>
     </PageContainer>

app/src/hooks/customHooks.js

@@ -7,6 +7,7 @@ import {
     fetchDomains,
     filesSearch,
     getArchive,
+    getArchiveStatistics,
     getChannel,
     getConfigs,
     getDomain,
@@ -1151,6 +1152,29 @@ export const useVideoStatistics = () => {
     return {statistics, fetchStatistics}
 }
 
+export const useArchiveStatistics = () => {
+    const [statistics, setStatistics] = useState(null);
+
+    const fetchStatistics = async () => {
+        try {
+            let stats = await getArchiveStatistics();
+            stats.archives.sum_size = humanFileSize(stats.archives.sum_size);
+            stats.archives.max_size = humanFileSize(stats.archives.max_size);
+            stats.historical.average_size = humanFileSize(stats.historical.average_size);
+            setStatistics(stats);
+        } catch (e) {
+            console.error(e);
+            setStatistics(undefined); // Display error.
+        }
+    }
+
+    useEffect(() => {
+        fetchStatistics();
+    }, []);
+
+    return {statistics, fetchStatistics}
+}
+
 export const useInventory = (inventoryId) => {
     const [byCategory, setByCategory] = useState();
     const [bySubcategory, setBySubcategory] = useState();

modules/archive/api.py

@@ -24,6 +24,13 @@ archive_bp = Blueprint('Archive', '/api/archive')
 logger = logger.getChild(__name__)
 
 
+@archive_bp.get('/statistics')
+@openapi.response(HTTPStatus.OK, schema.ArchiveStatisticsResponse)
+async def statistics(_: Request):
+    ret = await lib.get_statistics()
+    return json_response(ret, HTTPStatus.OK)
+
+
 @archive_bp.get('/<archive_id:int>')
 @openapi.description('Get an archive')
 @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)

modules/archive/lib.py

@@ -367,7 +367,8 @@ def format_archive_filename(
     except KeyError as e:
         logger.error(f'Invalid variable in archive file_name_format: {e}')
         # Fallback to default format
-        return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
+        return trim_file_name(
+            f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
 
 
 def preview_archive_filename(file_name_format: str) -> str:
@@ -465,7 +466,8 @@ def format_archive_filename_from_archive(
     except KeyError as e:
         logger.error(f'Invalid variable in archive file_name_format: {e}')
         # Fallback to default format
-        return trim_file_name(f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
+        return trim_file_name(
+            f'{archive_strftime(download_date)}_{escape_file_name(title) if title else "untitled"}.html')
 
 
 @dataclass
@@ -1420,6 +1422,68 @@ async def singlefile_to_archive(singlefile: bytes, destination: pathlib.Path = N
     return archive
 
 
+async def get_statistics():
+    """Get statistics about Archives and Domain Collections."""
+    with get_db_curs() as curs:
+        curs.execute('''
+                     SELECT
+                         -- total archives
+                         COUNT(a.id)                                                                         AS "archives",
+                         -- total archives downloaded over the past week/month/year
+                         COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 week') AS "week",
+                         COUNT(a.id)
+                         FILTER (WHERE fg.download_datetime >= current_date - interval '1 month')            AS "month",
+                         COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 year') AS "year",
+                         -- sum of all archive file sizes
+                         COALESCE(SUM(fg.size), 0)::BIGINT                                                   AS "sum_size",
+                         -- largest archive
+                         COALESCE(MAX(fg.size), 0)                                                           AS "max_size"
+                     FROM archive a
+                              LEFT JOIN file_group fg on a.file_group_id = fg.id
+                     ''')
+        archive_stats = dict(curs.fetchone())
+
+        # Get the total archives downloaded every month for the past two years.
+        curs.execute('''
+                     SELECT DATE_TRUNC('month', months.a),
+                            COUNT(archive.id)::BIGINT,
+                            SUM(size)::BIGINT AS "size"
+                     FROM generate_series(
+                                  date_trunc('month', current_date) - interval '2 years',
+                                  date_trunc('month', current_date) - interval '1 month',
+                                  '1 month'::interval) AS months(a),
+                          archive
+                              LEFT JOIN file_group fg on archive.file_group_id = fg.id
+                     WHERE fg.download_datetime >= date_trunc('month', months.a)
+                       AND fg.download_datetime < date_trunc('month', months.a) + interval '1 month'
+                       AND fg.download_datetime IS NOT NULL
+                     GROUP BY 1
+                     ORDER BY 1
+                     ''')
+        monthly_archives = [dict(i) for i in curs.fetchall()]
+
+        historical_stats = dict(monthly_archives=monthly_archives)
+        historical_stats['average_count'] = (sum(i['count'] for i in monthly_archives) // len(monthly_archives)) \
+            if monthly_archives else 0
+        historical_stats['average_size'] = (sum(i['size'] for i in monthly_archives) // len(monthly_archives)) \
+            if monthly_archives else 0
+
+        curs.execute('''
+                     SELECT COUNT(c.id)                                         AS "domains",
+                            COUNT(c.id) FILTER ( WHERE col.tag_id IS NOT NULL ) AS "tagged_domains"
+                     FROM collection c
+                              LEFT JOIN collection col on col.id = c.id
+                     WHERE c.kind = 'domain'
+                     ''')
+        domain_stats = dict(curs.fetchone())
+    ret = dict(statistics=dict(
+        archives=archive_stats,
+        domains=domain_stats,
+        historical=historical_stats,
+    ))
+    return ret
+
+
 async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
     """
     Generate a screenshot for an existing Archive that doesn't have one.

modules/archive/schema.py

@@ -35,3 +35,25 @@ class ArchiveSearchResponse:
 @dataclass
 class ArchiveFileFormatRequest:
     archive_file_format: str
+
+
+@dataclass
+class ArchiveStatistics:
+    archives: int
+    week: int
+    month: int
+    year: int
+    sum_size: int
+    max_size: int
+
+
+@dataclass
+class DomainStatistics:
+    domains: int
+    tagged_domains: int
+
+
+@dataclass
+class ArchiveStatisticsResponse:
+    archives: ArchiveStatistics
+    domains: DomainStatistics

modules/archive/test/test_lib.py

@@ -1357,3 +1357,36 @@ def test_format_archive_filename_escapes_special_chars(test_directory, fake_now)
     assert '/' not in result.split('/')[-1]  # No slashes in filename part
     assert '2000-01-01' in result
     assert result.endswith('.html')
+
+
+@pytest.mark.asyncio
+async def test_get_statistics(test_session, archive_factory):
+    """get_statistics returns archive and domain statistics."""
+    from modules.archive.lib import get_statistics
+
+    # Can get statistics in empty DB.
+    result = await get_statistics()
+    assert 'statistics' in result
+    assert 'archives' in result['statistics']
+    assert 'domains' in result['statistics']
+    assert 'historical' in result['statistics']
+
+    # Empty stats should have zero values.
+    stats = result['statistics']
+    assert stats['archives']['archives'] == 0
+    assert stats['domains']['domains'] == 0
+
+    # Create some archives.
+    archive1 = archive_factory('example.com')
+    archive2 = archive_factory('example.com')
+    archive3 = archive_factory('example.org')
+    archive4 = archive_factory()  # No domain
+    test_session.commit()
+
+    result = await get_statistics()
+    stats = result['statistics']
+
+    # Should count all 4 archives.
+    assert stats['archives']['archives'] == 4
+    # Should count 2 domain collections (example.com and example.org).
+    assert stats['domains']['domains'] == 2