User can select from/to years when searching files. Indexing doc and docx files.

2026-10-25 18:40:06 -04:00 · 2024-03-06 17:41:04 -07:00 · 2024-03-06 17:41:04 -07:00 · 00fe799b8f
commit 00fe799b8f
parent 758ff8e810
40 changed files with 880 additions and 159 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -73,7 +73,7 @@ jobs:
    steps:
      - checkout
      - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
      - restore_cache:
          key: deps-3.8-{{ checksum "requirements.txt" }}-2
      - run:
@ -100,7 +100,7 @@ jobs:
    steps:
      - checkout
      - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
      - restore_cache:
          key: deps-3.9-{{ checksum "requirements.txt" }}-2
      - run:
@ -127,7 +127,7 @@ jobs:
    steps:
      - checkout
      - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
      - restore_cache:
          key: deps-3.10-{{ checksum "requirements.txt" }}-2
      - run:
@ -154,7 +154,7 @@ jobs:
    steps:
      - checkout
      - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
      - restore_cache:
          key: deps-3.11-{{ checksum "requirements.txt" }}-2
      - run:
@ -181,7 +181,7 @@ jobs:
    steps:
      - checkout
      - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
      - restore_cache:
          key: deps-3.12-{{ checksum "requirements.txt" }}-2
      - run:
--- a/alembic/versions/1c041aeee0fc_doc_indexer.py
+++ b/alembic/versions/1c041aeee0fc_doc_indexer.py
@ -0,0 +1,36 @@
+"""Doc indexer.
+
+Revision ID: 1c041aeee0fc
+Revises: d96a04eb2785
+Create Date: 2024-03-05 16:54:34.478974
+
+"""
+import os
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = '1c041aeee0fc'
+down_revision = 'd96a04eb2785'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    # Re-index Word documents.
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    session.execute(
+        """
+        UPDATE file_group
+        SET indexed=false
+        WHERE mimetype in
+            ('application/msword','application/vnd.openxmlformats-officedocument.wordprocessingml.document')
+   """)
+
+
+def downgrade():
+    pass
--- a/app/src/App.js
+++ b/app/src/App.js
@ -10,7 +10,7 @@ import {MoreRoute} from "./components/Apps";
 import {InventoryRoute} from "./components/Inventory";
 import {ArchiveRoute} from "./components/Archive";
 import {FilesRoute} from "./components/Files";
-import {StatusProvider} from "./hooks/customHooks";
+import {QueryProvider, StatusProvider} from "./hooks/customHooks";
 import {MapRoute} from "./components/Map";
 import {MediaContextProvider, mediaStyles, StatusContext, ThemeContext} from "./contexts/contexts";
 import {Header, ThemeProvider} from "./components/Theme";
@ -61,17 +61,19 @@ function HelpPage() {
 }

 function Root() {
-    return <ThemeProvider>
-        <TagsProvider>
-            <FilePreviewProvider>
-                <header>
-                    <NavBar/>
-                </header>
-                <Outlet/>
-                <Footer/>
-            </FilePreviewProvider>
-        </TagsProvider>
-    </ThemeProvider>
+    return <QueryProvider>
+        <ThemeProvider>
+            <TagsProvider>
+                <FilePreviewProvider>
+                    <header>
+                        <NavBar/>
+                    </header>
+                    <Outlet/>
+                    <Footer/>
+                </FilePreviewProvider>
+            </TagsProvider>
+        </ThemeProvider>
+    </QueryProvider>
 }

 const router = createBrowserRouter(createRoutesFromElements(<Route
--- a/app/src/DashboardPage.js
+++ b/app/src/DashboardPage.js
@ -26,7 +26,7 @@ import {SearchView, useSearch, useSearchSuggestions} from "./components/Search";
 import {KiwixRestartMessage, OutdatedZimsMessage} from "./components/Zim";
 import {useWROLMode} from "./hooks/customHooks";
 import {FileSearchFilterButton} from "./components/Files";
-import {FilePreviewContext} from "./components/FilePreview";
+import {dateRangeIsEmpty, DateSelectorButton} from "./components/DatesSelector";

 function FlagsMessages({flags}) {
    const {settings, fetchSettings} = React.useContext(SettingsContext);
@ -170,11 +170,11 @@ export function DashboardPage() {
        loading,
        setSearchStr: setSuggestionSearchStr,
        setSearchTags,
+        months, setMonths,
+        dateRange, setDateRange,
    } = useSearchSuggestions(searchStr, activeTags);
    const {status} = useContext(StatusContext);

-    const {setPreviewFile} = React.useContext(FilePreviewContext);
-
    React.useEffect(() => {
        setSuggestionSearchStr(localSearchStr);
    }, [localSearchStr]);
@ -231,24 +231,53 @@ export function DashboardPage() {
        <Media at='mobile'>
            <Grid>
                <Grid.Row columns={2}>
-                    <Grid.Column width={13}>
-                        {getSearchResultsInput({size: 'large'})}
+                    <Grid.Column width={12}>
+                        {getSearchResultsInput()}
                    </Grid.Column>
-                    <Grid.Column width={3} textAlign='right'>
-                        <FileSearchFilterButton size='large'/>
+                    <Grid.Column width={1} textAlign='right' style={{padding: 0}}>
+                        <DateSelectorButton onMonthsChange={setMonths} defaultMonthsSelected={months}
+                                            defaultDateRange={dateRange}
+                                            onDateRangeChange={setDateRange}/>
+                    </Grid.Column>
+                    <Grid.Column width={1} textAlign='right'>
+                        <FileSearchFilterButton/>
                    </Grid.Column>
                </Grid.Row>
            </Grid>
        </Media>
-        <Media greaterThanOrEqual='tablet'>
+        <Media at='tablet'>
            <Grid>
                <Grid.Row columns={2}>
-                    <Grid.Column mobile={14}>
-                        {getSearchResultsInput({size: 'big'})}
+                    <Grid.Column textAlign='right' width={2}>
+                        <DateSelectorButton onMonthsChange={setMonths} defaultMonthsSelected={months}
+                                            defaultDateRange={dateRange}
+                                            onDateRangeChange={setDateRange}
+                                            buttonProps={{size: 'big'}}/>
                    </Grid.Column>
                    <Grid.Column textAlign='right' width={2}>
                        <FileSearchFilterButton size='big'/>
                    </Grid.Column>
+                    <Grid.Column mobile={12}>
+                        {getSearchResultsInput({size: 'big'})}
+                    </Grid.Column>
+                </Grid.Row>
+            </Grid>
+        </Media>
+        <Media greaterThanOrEqual='computer'>
+            <Grid>
+                <Grid.Row columns={2}>
+                    <Grid.Column textAlign='right' width={1}>
+                        <DateSelectorButton onMonthsChange={setMonths} defaultMonthsSelected={months}
+                                            defaultDateRange={dateRange}
+                                            onDateRangeChange={setDateRange}
+                                            buttonProps={{size: 'big'}}/>
+                    </Grid.Column>
+                    <Grid.Column textAlign='right' width={1}>
+                        <FileSearchFilterButton size='big'/>
+                    </Grid.Column>
+                    <Grid.Column mobile={14}>
+                        {getSearchResultsInput({size: 'big'})}
+                    </Grid.Column>
                </Grid.Row>
            </Grid>
        </Media>
--- a/app/src/api.js
+++ b/app/src/api.js
@ -1,5 +1,6 @@
 import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API, ZIM_API} from "./components/Common";
 import {toast} from "react-semantic-toasts-2";
+import _ from "lodash";

 function timeoutPromise(ms, promise) {
    // Create a timeout wrapper around a promise.  If the timeout is reached, throw an error.  Otherwise, return
@ -545,7 +546,7 @@ export async function deleteDownload(downloadId) {
    }
 }

-export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline) {
+export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear) {
    const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
    if (mimetypes) {
        body['mimetypes'] = mimetypes;
@ -559,7 +560,16 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta
    if (headline) {
        body['headline'] = true;
    }
-    console.debug('searching files', body);
+    if (!_.isEmpty(months)) {
+        body['months'] = months.map(i => parseInt(i));
+    }
+    if (fromYear) {
+        body['from_year'] = fromYear;
+    }
+    if (toYear) {
+        body['to_year'] = toYear;
+    }
+    console.info('searching files', body);
    const response = await apiPost(`${API_URI}/files/search`, body);

    if (response.status === 200) {
@ -1050,8 +1060,14 @@ export async function fetchDecoded(vinNumber) {
    }
 }

-export async function searchSuggestions(search_str, tagNames, mimetypes) {
-    const body = {search_str: search_str, tag_names: tagNames, mimetypes: mimetypes};
+export async function searchSuggestions(search_str, tagNames, mimetypes, months, dateRange) {
+    months = months ? months.map(i => parseInt(i)) : [];
+
+    const body = {search_str, tag_names: tagNames, mimetypes, months};
+    if (dateRange) {
+        body['from_year'] = dateRange[0];
+        body['to_year'] = dateRange[1];
+    }
    const response = await apiPost(`${API_URI}/search_suggestions`, body);
    if (response.ok) {
        const content = await response.json();
--- a/app/src/components/Common.js
+++ b/app/src/components/Common.js
@ -1657,7 +1657,22 @@ export function getParentDirectory(filePath) {
 export function MultilineText({text, ...props}) {
    return <div {...props}>
        {text.split('\n').map((line, index, array) =>
-            index === array.length - 1 ? line : <p>{line}</p>
+            index === array.length - 1 ? line : <p key={index}>{line}</p>
        )}
    </div>
 }
+
+export const monthNames = [
+    'January',
+    'February',
+    'March',
+    'April',
+    'May',
+    'June',
+    'July',
+    'August',
+    'September',
+    'October',
+    'November',
+    'December',
+];
--- a/app/src/components/DatesSelector.js
+++ b/app/src/components/DatesSelector.js
@ -0,0 +1,230 @@
+import React from "react";
+import {Button, Form, FormField, Modal, ModalActions, ModalContent, ModalHeader, Segment} from "./Theme";
+import {Checkbox, Dropdown} from "semantic-ui-react";
+import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
+import {monthNames} from "./Common";
+import Message from "semantic-ui-react/dist/commonjs/collections/Message";
+
+export const dateRangeIsEmpty = (dateRange) => {
+    return dateRange[0] === null && dateRange[1] === null;
+}
+
+function MonthsForm({monthsSelected, setMonthsSelected}) {
+    monthsSelected = monthsSelected.map(i => parseInt(i));
+
+    const handleWinter = (e) => {
+        if (e) e.preventDefault();
+        setMonthsSelected([12, 1, 2]);
+    }
+
+    const handleSpring = (e) => {
+        if (e) e.preventDefault();
+        setMonthsSelected([3, 4, 5]);
+    }
+
+    const handleSummer = (e) => {
+        if (e) e.preventDefault();
+        setMonthsSelected([6, 7, 8]);
+    }
+
+    const handleFall = (e) => {
+        if (e) e.preventDefault();
+        setMonthsSelected([9, 10, 11]);
+    }
+
+    const monthCheckbox = (label) => {
+        // "January" == 1, etc.
+        const idx = monthNames.indexOf(label) + 1;
+        const isChecked = monthsSelected.indexOf(idx) >= 0;
+        return <Checkbox
+            label={label}
+            checked={isChecked}
+            size='large'
+            onChange={(e, {checked}) => {
+                // Add or remove month index if the respective checkbox is checked.
+                if (checked) {
+                    setMonthsSelected([...monthsSelected, idx]);
+                } else {
+                    setMonthsSelected(monthsSelected.filter(i => i !== idx));
+                }
+            }}
+        />
+    }
+
+    return <Form>
+        <Grid>
+            <Grid.Row columns={3}>
+                <Grid.Column>{monthCheckbox('January')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('February')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('March')}</Grid.Column>
+            </Grid.Row>
+            <Grid.Row columns={3}>
+                <Grid.Column>{monthCheckbox('April')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('May')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('June')}</Grid.Column>
+            </Grid.Row>
+            <Grid.Row columns={3}>
+                <Grid.Column>{monthCheckbox('July')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('August')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('September')}</Grid.Column>
+            </Grid.Row>
+            <Grid.Row columns={3}>
+                <Grid.Column>{monthCheckbox('October')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('November')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('December')}</Grid.Column>
+            </Grid.Row>
+            <Grid.Row columns={4}>
+                <Grid.Column><Button onClick={handleWinter} color='blue' size='small'>Winter</Button></Grid.Column>
+                <Grid.Column><Button onClick={handleSpring} color='green' size='small'>Spring</Button></Grid.Column>
+                <Grid.Column><Button onClick={handleSummer} color='yellow' size='small'>Summer</Button></Grid.Column>
+                <Grid.Column><Button onClick={handleFall} color='red' size='small'>Fall</Button></Grid.Column>
+            </Grid.Row>
+        </Grid>
+    </Form>
+}
+
+function DateRangeForm({dateRange, setDateRange}) {
+    const currentYear = (new Date()).getFullYear();
+    const [error, setError] = React.useState('');
+
+    let yearRange = [];
+    for (let i = 1970; i <= currentYear; i++) {
+        yearRange = [...yearRange, {
+            key: i,
+            text: i,
+            value: i,
+        }];
+    }
+
+    const handleFromYear = (e, {value}) => {
+        const toYear = dateRange[1] || currentYear;
+        setDateRange([value, toYear]);
+    }
+
+    const handleToYear = (e, {value}) => {
+        setDateRange([dateRange[0], value]);
+    }
+
+    React.useEffect(() => {
+        if (dateRange[0] > dateRange[1]) {
+            setError('From Year must be greater than To Year');
+        } else {
+            setError('');
+        }
+    }, [dateRange])
+
+    const errorMessage = error && <Message error content={error}/>;
+
+    return <React.Fragment>
+        <Form>
+            <Grid>
+                <Grid.Row columns={2}>
+                    <Grid.Column>
+                        <FormField>
+                            <label>From Year</label>
+                            <Dropdown search scrolling fluid
+                                      options={yearRange}
+                                      value={dateRange[0]}
+                                      onChange={handleFromYear}
+                                      error={!!error}
+                            />
+                        </FormField>
+                    </Grid.Column>
+                    <Grid.Column>
+                        <FormField>
+                            <label>To Year</label>
+                            <Dropdown search scrolling fluid
+                                      options={yearRange.reverse()}
+                                      value={dateRange[1]}
+                                      onChange={handleToYear}
+                                      error={!!error}
+                            />
+                        </FormField>
+                    </Grid.Column>
+                </Grid.Row>
+            </Grid>
+        </Form>
+        {errorMessage}
+    </React.Fragment>
+}
+
+export function DateSelectorButton({
+                                       onMonthsChange,
+                                       onDateRangeChange,
+                                       defaultMonthsSelected,
+                                       defaultDateRange,
+                                       buttonProps
+                                   }) {
+    const emptyDateRange = [null, null];
+
+    const [open, setOpen] = React.useState(false);
+    const [monthsSelected, setMonthsSelected] = React.useState(defaultMonthsSelected || []);
+    const [dateRange, setDateRange] = React.useState(defaultDateRange || emptyDateRange);
+    const [color, setColor] = React.useState('grey');
+
+    React.useEffect(() => {
+        if (monthsSelected.length > 0 || (!dateRangeIsEmpty(dateRange))) {
+            setColor('violet');
+        } else {
+            setColor('grey');
+        }
+    }, [JSON.stringify(dateRange), JSON.stringify(monthsSelected)]);
+
+    const handleOpen = (e) => {
+        if (e) e.preventDefault();
+        setOpen(true);
+    }
+
+    const handleClose = (e) => {
+        if (e) e.preventDefault();
+        setOpen(false);
+        // Only submit selection when user has closed selector.
+        if (onDateRangeChange && dateRange[0] <= dateRange[1]) {
+            onDateRangeChange(dateRange);
+        } else if (onDateRangeChange && dateRangeIsEmpty(dateRange)) {
+            //Clear the date range.
+            onDateRangeChange(dateRange);
+        }
+        if (onMonthsChange) {
+            onMonthsChange(monthsSelected);
+        }
+    }
+
+    const localOnSetDateRange = (newDateRange) => {
+        setDateRange(newDateRange);
+    }
+
+    const handleClear = (e) => {
+        if (e) e.preventDefault();
+        setDateRange(emptyDateRange);
+        setMonthsSelected([]);
+    }
+
+    return <React.Fragment>
+        <Button
+            icon='calendar alternate'
+            onClick={handleOpen}
+            color={color}
+            {...buttonProps}
+        />
+        <Modal closeIcon
+               open={open}
+               onClose={handleClose}
+               size='tiny'
+        >
+            <ModalHeader>Filter by Date</ModalHeader>
+            <ModalContent>
+                <Segment>
+                    <DateRangeForm dateRange={dateRange} setDateRange={localOnSetDateRange}/>
+                </Segment>
+                <Segment>
+                    <MonthsForm monthsSelected={monthsSelected} setMonthsSelected={setMonthsSelected}/>
+                </Segment>
+            </ModalContent>
+            <ModalActions>
+                <Button onClick={handleClear} secondary>Clear</Button>
+                <Button onClick={handleClose}>Close</Button>
+            </ModalActions>
+        </Modal>
+    </React.Fragment>
+}
--- a/app/src/components/FilePreview.js
+++ b/app/src/components/FilePreview.js
@ -194,7 +194,7 @@ export function FilePreviewProvider({children}) {
        if (previewQuery && !previewFile) {
            initPreviewFile();
        }
-    }, []);
+    }, [previewQuery]);

    const localFetchFile = async () => {
        // Get the file again with its Tags.
--- a/app/src/components/Files.js
+++ b/app/src/components/Files.js
@ -32,9 +32,9 @@ import {
    useTitle
 } from "./Common";
 import {
+    useAllQuery,
    useFilesProgressInterval,
    usePages,
-    useQuery,
    useSearchFiles,
    useSearchFilter,
    useSearchView,
@ -149,7 +149,7 @@ function FileCard({file}) {
                </ExternalCardLink>
            </CardHeader>
            {author && <b {...s}>{author}</b>}
-            <p>{isoDatetimeToString(file.modified)}</p>
+            <p>{isoDatetimeToString(file.published_datetime) || isoDatetimeToString(file.modified)}</p>
            <p>{size}</p>
        </CardContent>
    </Card>
@ -403,6 +403,9 @@ export function SearchFilter({filters = [], modalHeader, size = 'medium'}) {
        </Form.Field>
    );

+    // Use violet color when filter has been applied.
+    const buttonColor = filter ? 'violet' : 'grey';
+
    if (filters && filters.length > 0) {
        return <>
            <Modal open={open} onOpen={() => handleOpen()} onClose={() => setOpen(false)} closeIcon>
@ -420,7 +423,7 @@ export function SearchFilter({filters = [], modalHeader, size = 'medium'}) {
            <Button
                icon='filter'
                onClick={handleOpen}
-                primary={!!filter}
+                color={buttonColor}
                size={size}
            />
        </>
@ -430,12 +433,11 @@ export function SearchFilter({filters = [], modalHeader, size = 'medium'}) {
 }

 export function TagsQuerySelector({onChange}) {
-    // Creates  dropdown that the User can use to manipulate the tag query.
-    const {searchParams, updateQuery} = useQuery();
-    const activeTags = searchParams.getAll('tag');
+    // Creates modal that the User can use to manipulate the tag query.
+    const [activeTags, setActiveTags] = useAllQuery('tag');

    const localOnChange = (tagNames) => {
-        updateQuery({'tag': tagNames});
+        setActiveTags(tagNames);
        if (onChange) {
            onChange(tagNames);
        }
--- a/app/src/components/Search.js
+++ b/app/src/components/Search.js
@ -1,7 +1,7 @@
 import React, {useState} from "react";
-import {createSearchParams, Route, Routes, useNavigate} from "react-router-dom";
+import {Route, Routes, useNavigate} from "react-router-dom";
 import {FilesSearchView} from "./Files";
-import {useLatestRequest, usePages, useQuery, useSearchFilter} from "../hooks/customHooks";
+import {useLatestRequest, usePages, useSearchDateRange, useSearchFilter, useSearchMonths} from "../hooks/customHooks";
 import {ZimSearchView} from "./Zim";
 import {searchEstimateZims, searchSuggestions} from "../api";
 import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common";
@ -9,6 +9,7 @@ import _ from "lodash";
 import {TagsContext} from "../Tags";
 import {Button as SButton, Header as SHeader, Label} from "semantic-ui-react";
 import {Modal, ModalContent} from "./Theme";
+import {QueryContext} from "../contexts/contexts";

 const SUGGESTED_APPS = [
    {location: '/admin', title: 'Downloads', description: 'View and control your downloads'},
@ -37,7 +38,9 @@ const SUGGESTED_APPS = [
 export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false, model) => {
    const navigate = useNavigate();

-    const {searchParams, updateQuery} = useQuery();
+    const {months, setMonths} = useSearchMonths();
+    const {dateRange, setDateRange} = useSearchDateRange();
+    const {searchParams, updateQuery, getLocationStr} = React.useContext(QueryContext);
    // `searchStr` means actually fetch the files/zims.
    const searchStr = searchParams.get('q');
    // User can search only by Tags, `searchStr` not required.
@ -53,11 +56,8 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
        if (filter) {
            searchQuery['filter'] = filter;
        }
-        navigate({
-            pathname: '/search',
-            // Start new search at offset 0.
-            search: createSearchParams(searchQuery).toString(),
-        });
+        const location = getLocationStr(searchQuery, '/search');
+        navigate(location);
    }

    const clearSearch = () => {
@ -90,6 +90,8 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
        setSearchStr,
        clearSearch,
        setTags,
+        months, setMonths,
+        dateRange, setDateRange,
    }
 }

@ -101,6 +103,8 @@ export function useSuggestions(searchStr, tagNames, filter) {
        zimsEstimates: [],
    }
    const [suggestions, setSuggestions] = React.useState(defaultSuggestions);
+    const {months} = useSearchMonths();
+    const {dateRange} = useSearchDateRange();
    // fileGroups/channels/domains.
    const {data, sendRequest, loading} = useLatestRequest(500);
    // Zims are slow, so they are separate.
@ -109,10 +113,17 @@ export function useSuggestions(searchStr, tagNames, filter) {
    React.useEffect(() => {
        if ((searchStr && searchStr.length > 0) || (tagNames && tagNames.length > 0)) {
            const mimetypes = filterToMimetypes(filter);
-            sendRequest(async () => await searchSuggestions(searchStr, tagNames, mimetypes));
+            sendRequest(async () => await searchSuggestions(searchStr, tagNames, mimetypes, months, dateRange));
            sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
        }
-    }, [searchStr, sendRequest, sendZimRequest, JSON.stringify(tagNames)]);
+    }, [
+        searchStr,
+        sendRequest,
+        sendZimRequest,
+        JSON.stringify(tagNames),
+        JSON.stringify(months),
+        JSON.stringify(dateRange),
+    ]);

    React.useEffect(() => {
        if (!_.isEmpty(data)) {
@ -141,8 +152,10 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames) {
    const [searchStr, setSearchStr] = React.useState(defaultSearchStr || '');
    const [searchTags, setSearchTags] = React.useState(defaultTagNames);
    const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
+    const {months, setMonths} = useSearchMonths();
+    const {dateRange, setDateRange} = useSearchDateRange();
    const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter);
-    const {getLocationStr} = useQuery();
+    const {getLocationStr} = React.useContext(QueryContext);

    // The results that will be displayed by <Search>.
    const [suggestionsResults, setSuggestionsResults] = useState({});
@ -275,12 +288,13 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames) {
        suggestions,
        suggestionsResults,
        suggestionsSums,
-        searchStr,
-        setSearchStr,
+        searchStr, setSearchStr,
        setSearchTags,
+        months, setMonths,
        handleResultSelect,
        resultRenderer,
        loading,
+        dateRange, setDateRange,
    }
 }

--- a/app/src/components/Videos.js
+++ b/app/src/components/Videos.js
@ -52,7 +52,7 @@ import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
 import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
 import {Button, Card, Header, Loader, Placeholder, Segment, Statistic, StatisticGroup} from "./Theme";
 import {deleteVideos} from "../api";
-import {Media, ThemeContext} from "../contexts/contexts";
+import {Media, QueryContext, ThemeContext} from "../contexts/contexts";
 import _ from "lodash";

 export function VideoWrapper() {
@ -69,7 +69,7 @@ export function VideoWrapper() {
 function VideosPage() {

    const {channelId} = useParams();
-    const {searchParams} = useQuery();
+    const {searchParams} = React.useContext(QueryContext);
    const [selectedVideos, setSelectedVideos] = useState([]);

    let searchOrder = defaultVideoOrder;
--- a/app/src/contexts/contexts.js
+++ b/app/src/contexts/contexts.js
@ -13,6 +13,13 @@ export const SettingsContext = React.createContext({
    fetchSettings: null,
 });

+export const QueryContext = React.createContext({
+    searchParams: null,
+    setSearchParams: null,
+    updateQuery: null,
+    getLocationStr: null,
+})
+
 export const AppMedia = createMedia({
    breakpoints: {
        mobile: 0, tablet: 700, computer: 1024,
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@ -15,7 +15,6 @@ import {
    getSettings,
    getStatistics,
    getStatus,
-    getThrottleStatus,
    getVideo,
    getVideosStatistics,
    searchArchives,
@ -28,7 +27,7 @@ import {
 } from "../api";
 import {createSearchParams, useLocation, useSearchParams} from "react-router-dom";
 import {enumerate, filterToMimetypes, humanFileSize, secondsToFullDuration} from "../components/Common";
-import {SettingsContext, StatusContext} from "../contexts/contexts";
+import {QueryContext, SettingsContext, StatusContext} from "../contexts/contexts";
 import {toast} from "react-semantic-toasts-2";
 import {useSearch} from "../components/Search";
 import _ from "lodash";
@ -103,63 +102,95 @@ export const useLatestRequest = (delay = 300, defaultLoading = false) => {
    return {data, sendRequest, loading};
 };

+const getSearchParamCopy = (searchParams) => {
+    let copy = {};
+    if (searchParams) {
+        Array.from(searchParams.keys()).forEach(key => {
+            const value = searchParams.getAll(key)
+            copy[key] = value.length === 1 ? value[0] : value;
+        })
+    }
+    copy = removeEmptyValues(copy);
+    return copy;
+}

-export const useQuery = () => {
+const removeEmptyValues = (obj) => {
+    Object.entries(obj).forEach(([k, v]) => {
+        // Clear any empty values.
+        if (v === null || v === undefined || (Array.isArray(v) && v.length === 0)) {
+            delete obj[k];
+        }
+    });
+    return obj
+}
+
+
+export const useQuery = (defaultParams) => {
    const location = useLocation();
    const [searchParams, setSearchParams] = useSearchParams();
+    const [state, setState] = React.useState(defaultParams || getSearchParamCopy(searchParams));

-    const setQuery = (obj, replace = true) => {
-        console.debug(`setQuery`, obj);
-        setSearchParams(createSearchParams(obj), {replace: replace});
-    }
-
-    const getQuery = (obj) => {
-        const newQuery = {};
-        for (const entry of searchParams.entries()) {
-            newQuery[entry[0]] = entry[1];
+    React.useEffect(() => {
+        // Copy the state into the URL.
+        setSearchParams(state);
+        if (!_.isEmpty(state)) {
+            console.debug('Changing URL params', state);
        }
-        Object.entries(obj).forEach(([key, value]) => {
-            if (value === undefined || value === null || value === '') {
-                delete newQuery[key];
-            } else {
-                newQuery[key] = value;
-            }
-        })
-        return newQuery;
-    }
+    }, [state]);

-    const updateQuery = (obj) => {
-        const newQuery = getQuery(obj);
-        setQuery(newQuery);
-    }
+    React.useEffect(() => {
+        // Keep URL and state in sync.
+        setState(getSearchParamCopy(searchParams));
+    }, [searchParams]);
+
+    const updateQuery = (newParams, replace = false) => {
+        // Update the old state with the new values.
+        setState(oldState => {
+            if (replace) {
+                // Ignore oldState because all values are going to be replaced.
+                return removeEmptyValues(newParams);
+            } else {
+                return removeEmptyValues({...oldState, ...newParams});
+            }
+        });
+    };

    const getLocationStr = (newSearchParams, pathname) => {
        // Get the current location, but with new params appended.
-        const newQuery = createSearchParams(getQuery(newSearchParams));
+        newSearchParams = {...state, ...newSearchParams};
+        const newQuery = createSearchParams(newSearchParams);
        return `${pathname || location.pathname}?${newQuery.toString()}`
    }

-    return {searchParams, setSearchParams, setQuery, updateQuery, getLocationStr}
+    return {searchParams, updateQuery, getLocationStr}
+}
+
+
+export const QueryProvider = (props) => {
+    const value = useQuery();
+
+    return <QueryContext.Provider value={value}>
+        {props.children}
+    </QueryContext.Provider>
 }

 export const useOneQuery = (name) => {
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
    const value = searchParams.get(name);

-    const setValue = (newValue) => {
-        console.debug(`useOneQuery setValue=${newValue}`);
-        updateQuery({[name]: newValue});
+    const setValue = (newValue, replace = false) => {
+        updateQuery({[name]: newValue}, replace);
    }

    return [value, setValue]
 }

 export const useAllQuery = (name) => {
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
    const value = searchParams.getAll(name);

-    const setValue = (newValue) => {
-        updateQuery({[name]: newValue});
+    const setValue = (newValue, replace = false) => {
+        updateQuery({[name]: newValue}, replace);
    }

    return [value, setValue]
@ -212,7 +243,7 @@ export const useArchive = (archiveId) => {
 }

 export const usePages = (defaultLimit = 24, totalPages = 0) => {
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
    const offset = searchParams.get('o') || 0;
    const limit = parseInt(searchParams.get('l') || defaultLimit || 24);
    const [activePage, setActivePage] = useState(calculatePage(offset, limit));
@ -232,7 +263,7 @@ export const usePages = (defaultLimit = 24, totalPages = 0) => {

    const setTotal = (total) => {
        const newTotalPages = calculateTotalPages(total, limit);
-        console.log('newTotalPages', newTotalPages);
+        console.debug('newTotalPages', newTotalPages);
        setTotalPages(newTotalPages);
    }

@ -245,7 +276,7 @@ export const usePages = (defaultLimit = 24, totalPages = 0) => {
 export const useSearchArchives = (defaultLimit) => {
    const {domain} = useSearchDomain();
    const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
    const searchStr = searchParams.get('q') || '';
    const order = searchParams.get('order');
    const activeTags = searchParams.getAll('tag');
@ -304,7 +335,7 @@ export const useSearchArchives = (defaultLimit) => {
 }

 export const useSearchVideos = (defaultLimit, channelId, order_by) => {
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
    const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
    const searchStr = searchParams.get('q') || '';
    const order = searchParams.get('order') || order_by;
@ -473,7 +504,9 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
        searchStr,
        filter,
        model: model_,
-        setSearchStr
+        setSearchStr,
+        months,
+        dateRange,
    } = useSearch(defaultLimit, emptySearch, model);
    const {view} = useSearchView();

@ -486,9 +519,16 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
        }
        const mimetypes = filterToMimetypes(filter);
        setSearchFiles(null);
+        let fromYear;
+        let toYear;
+        if (dateRange) {
+            fromYear = dateRange[0];
+            toYear = dateRange[1];
+        }
        try {
            let [file_groups, total] = await filesSearch(
-                pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline);
+                pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline,
+                months, fromYear, toYear);
            setSearchFiles(file_groups);
            pages.setTotal(total);
        } catch (e) {
@ -514,7 +554,17 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
        }
        // Handle when this is unmounted.
        return () => debouncedLocalSearchFiles.cancel();
-    }, [searchStr, pages.effect, filter, model, model_, JSON.stringify(activeTags), headline]);
+    }, [
+        searchStr,
+        pages.effect,
+        filter,
+        model,
+        model_,
+        JSON.stringify(activeTags),
+        headline,
+        JSON.stringify(months),
+        JSON.stringify(dateRange),
+    ]);

    return {
        searchFiles,
@ -933,6 +983,33 @@ export const useSearchOrder = () => {
    return {sort, setSort}
 }

+export const useSearchMonths = () => {
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
+    const months = searchParams.getAll('month');
+
+    const setMonths = (newMonths) => {
+        // Set new months, go back to first page.
+        updateQuery({'month': newMonths, 'o': 0});
+    }
+
+    return {months, setMonths}
+}
+
+export const useSearchDateRange = () => {
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
+    let fromDate = searchParams.get('fromDate');
+    let toDate = searchParams.get('toDate');
+    fromDate = fromDate ? parseInt(fromDate) : null;
+    toDate = toDate ? parseInt(toDate) : null;
+
+    const setDateRange = ([newFromDate, newToDate]) => {
+        // Set new date range, go back to first page.
+        updateQuery({fromDate: newFromDate, toDate: newToDate, 'o': 0});
+    }
+
+    return {dateRange: [fromDate, toDate], setDateRange}
+}
+
 export const useUploadFile = () => {
    const [files, setFiles] = useState([]);
    const [progresses, setProgresses] = useState({});
@ -983,7 +1060,7 @@ export const useUploadFile = () => {
                    handleProgress(file.name, chunkNum, totalChunks, 'pending', file.type);
                    const expectedChunk = data['expected_chunk'];
                    if (xhr.status === 416) {
-                        console.log(`Server requested a different chunk ${chunkNum}`);
+                        console.warn(`Server requested a different chunk ${chunkNum}`);
                        await uploadChunk(file, expectedChunk, chunkSize, totalChunks, tries + 1, maxTries);
                    } else {
                        console.debug(`Uploading of chunk ${chunkNum} succeeded, got request for chunk ${chunkNum}`);
@ -992,7 +1069,7 @@ export const useUploadFile = () => {
                    }
                } else if (xhr.status === 201) {
                    handleProgress(file.name, totalChunks, totalChunks, 'complete', file.type);
-                    console.log(`Uploading of ${file.path} completed.`);
+                    console.info(`Uploading of ${file.path} completed.`);
                } else if (xhr.status === 400) {
                    handleProgress(file.name, totalChunks, totalChunks, 'conflicting', file.type);
                    const data = JSON.parse(xhr.responseText);
@ -1046,7 +1123,7 @@ export const useUploadFile = () => {

 export const useSearchZims = (defaultLimit) => {
    const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
    const searchStr = searchParams.get('q') || '';

    const [zims, setZims] = useState(null);
--- a/debian-live-config/config/package-lists/wrolpi.list.chroot
+++ b/debian-live-config/config/package-lists/wrolpi.list.chroot
@ -8,6 +8,7 @@ bzip2
 ca-certificates
 calibre
 cargo
+catdoc
 chromium
 cpufrequtils
 curl
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,8 +1,7 @@
 # Development containers, these should only be used if you are developing.
 version: '3'
 services:
-  db:
-    # https://github.com/docker-library/docs/tree/master/postgres
+  db: # https://github.com/docker-library/docs/tree/master/postgres
    image: postgres:12
    environment:
      - POSTGRES_DB=wrolpi
@ -11,6 +10,7 @@ services:
      - "5432:5432"
    healthcheck:
      test: [ 'CMD-SHELL', 'pg_isready -U postgres' ]
+      interval: 10s

  api:
    depends_on:
@ -30,10 +30,11 @@ services:
      - ${REACT_APP_API-127.0.0.1:8081}:8081
    command: '-vv api --host 0.0.0.0'
    user: '${UID-1000}:${GID-1000}'
-  #    healthcheck:
-  #      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]
+      interval: 2m

-  archive:
+  archive: # Python/Sanic wrapper around Singlefile and Readability.
    build:
      context: .
      dockerfile: docker/archive/Dockerfile
@ -43,8 +44,9 @@ services:
      - '8083:8080'
    healthcheck:
      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8080/' ]
+      interval: 2m

-  app:
+  app: # React App
    depends_on:
      - api
    build:
@ -59,13 +61,13 @@ services:
    stdin_open: true
    healthcheck:
      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:3000' ]
+      interval: 1m
    user: '${UID-1000}:${GID-1000}'
    environment:
      # Force React to use nginx port to connect to it's websocket.
      - WDS_SOCKET_PORT=0

-  map:
-    # https://github.com/Overv/openstreetmap-tile-server
+  map: # https://github.com/Overv/openstreetmap-tile-server
    image: overv/openstreetmap-tile-server:v1.6.0
    volumes:
      - 'openstreetmap-data:/var/lib/postgresql/12/main'
@ -77,6 +79,9 @@ services:
      - '8084:80'
    command: 'run'
    shm_size: 1g  # Increase shared size to render large map tiles
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
+      interval: 1m

  web:
    depends_on:
@ -91,8 +96,9 @@ services:
      - './test:/opt/media'
    healthcheck:
      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
+      interval: 1m

-  zim:
+  zim: # kiwix-serve
    build:
      context: .
      dockerfile: docker/zim/Dockerfile
@ -101,6 +107,9 @@ services:
      - './docker/zim/entrypoint.sh:/entrypoint.sh'
    ports:
      - '8085:8085'
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8085' ]
+      interval: 1m

  help: # The WROLPi Help documentation.
    build:
@ -110,6 +119,9 @@ services:
      - './docker/help/wrolpi-help:/opt/wrolpi-help'
    ports:
      - '8086:8086'
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8086' ]
+      interval: 1m

 volumes:
  openstreetmap-data:
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@ -5,7 +5,7 @@ WORKDIR /opt/wrolpi
 # Update dependencies for the services install.
 RUN apt update
 # ffmpeg for Videos
-RUN apt-get install -y ffmpeg
+RUN apt-get install -y ffmpeg catdoc
 RUN ffmpeg -version

 # Install WROLPi
--- a/main.py
+++ b/main.py
@ -209,6 +209,9 @@ async def startup(app: Sanic):
            # We can't search for Videos missing comments until the refresh has completed.
            pass

+        # Wait for download manager to startup.
+        await asyncio.sleep(5)
+
        while True:
            # Fetch comments for videos every hour.
            if download_manager.disabled.is_set() or download_manager.stopped.is_set():
--- a/modules/map/lib.py
+++ b/modules/map/lib.py
@ -13,7 +13,7 @@ from wrolpi.common import get_media_directory, walk, logger
 from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp
 from wrolpi.db import optional_session, get_db_session
 from wrolpi.events import Events
-from wrolpi.vars import PROJECT_DIR
+from wrolpi.vars import PROJECT_DIR, IS_RPI5

 logger = logger.getChild(__name__)

@ -224,16 +224,42 @@ def get_import_status(session: Session = None) -> List[MapFile]:
    return map_paths


-# Bps calculated using many tests on a well-cooled RPi4.
+# Calculated using many tests on a well-cooled RPi4 (4GB).
 RPI4_PBF_BYTES_PER_SECOND = 61879
+RPI4_A = 7.17509261732342e-14
+RPI4_B = 6.6590760410412e-5
+RPI4_C = 10283
+# Calculated using many tests on a well-cooled RPi5 (8GB).
+RPI5_PBF_BYTES_PER_SECOND = 136003
+RPI5_A = 1.6681382703586e-15
+RPI5_B = 2.35907824676145e-6
+RPI5_C = 53.727
+
+
+def seconds_to_import_rpi4(size_in_bytes: int) -> int:
+    if size_in_bytes > 1_000_000_000:
+        # Use exponential curve for large files.
+        a = RPI4_A * size_in_bytes ** 2
+        b = RPI4_B * size_in_bytes
+        return int(a - b + RPI4_C)
+    # Use simpler equation for small files.
+    return max(int(size_in_bytes // RPI4_PBF_BYTES_PER_SECOND), 0)
+
+
+def seconds_to_import_rpi5(size_in_bytes: int) -> int:
+    if size_in_bytes > 5_000_000_000:
+        # Use exponential curve for large files.
+        a = RPI5_A * size_in_bytes ** 2
+        b = RPI5_B * size_in_bytes
+        return int(a - b + RPI5_C)
+    # Use simpler equation for small files.
+    return max(int(size_in_bytes // RPI5_PBF_BYTES_PER_SECOND), 0)


 def seconds_to_import(size_in_bytes: int) -> int:
    """Attempt to predict how long it will take an RPi4 to import a given PBF file."""
-    if size_in_bytes > 1_000_000_000:
-        # Use exponential curve for large files.  These magic numbers are from testing many imports on an RPi 4.
-        a = 7.17509261732342e-14 * size_in_bytes ** 2
-        b = 6.6590760410412e-5 * size_in_bytes
-        c = 10283
-        return int(a - b + c)
-    return max(int(size_in_bytes // RPI4_PBF_BYTES_PER_SECOND), 0)
+    if IS_RPI5:
+        return seconds_to_import_rpi5(size_in_bytes)
+
+    # Default to RPi4, because it's the most conservative estimate.
+    return seconds_to_import_rpi4(size_in_bytes)
--- a/modules/map/test/test_lib.py
+++ b/modules/map/test/test_lib.py
@ -1,3 +1,4 @@
+import unittest.mock
 from unittest import mock

 import pytest
@ -82,3 +83,21 @@ async def test_run_import_command(test_directory, mock_create_subprocess_shell):
 ])
 def test_seconds_to_import_rpi4(size, expected):
    assert lib.seconds_to_import(size) == expected
+
+
+@pytest.mark.parametrize('size,expected', [
+    (0, 0),
+    (-1, 0),
+    (17737381, 130),
+    (63434267, 466),
+    (87745484, 645),
+    (116318111, 855),
+    (136372996, 1002),
+    (1936075318, 14235),
+    (2676094489, 19676),
+    (3392375001, 24943),
+    (11346305075, 188040),
+])
+def test_seconds_to_import_rpi5(size, expected):
+    with unittest.mock.patch('modules.map.lib.IS_RPI5', True):
+        assert lib.seconds_to_import(size) == expected
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@ -207,8 +207,8 @@ async def search_channels_by_name(name: str, limit: int = 5, session: Session =
    name_no_spaces = ''.join(name.split(' '))
    stmt = session.query(Channel) \
        .filter(or_(
-        Channel.name.ilike(f'%{name}%'),
-        Channel.name.ilike(f'%{name_no_spaces}%'),
+            Channel.name.ilike(f'%{name}%'),
+            Channel.name.ilike(f'%{name_no_spaces}%'),
    )) \
        .order_by(asc(Channel.name)) \
        .limit(limit)
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@ -77,7 +77,6 @@ class ChannelDownloader(Downloader, ABC):
    @staticmethod
    def is_a_playlist(info: dict):
        # A playlist may have an id different from its channel.
-        logger.debug(f'is_a_playlist {info["id"]=} {info.get("channel_id")=}')
        return info['id'] != info.get('channel_id')

    async def do_download(self, download: Download) -> DownloadResult:
--- a/modules/videos/video/lib.py
+++ b/modules/videos/video/lib.py
@ -1,5 +1,4 @@
 import asyncio
-import json
 import random
 from datetime import timedelta
 from typing import Tuple, Optional, List, Dict
@ -218,6 +217,7 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
        with get_db_session(commit=True) as session_:
            for video_ in session_.query(Video).join(FileGroup).filter_by(url=video_url).all():
                video_.comments_failed = True
+        download_manager.add_to_skip_list(video_url)

    for url in video_urls:
        # Sleep to catch cancel.
@ -240,6 +240,7 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
                continue

        tries = 5
+        info = None
        while tries > 0:
            try:
                tries -= 1
@ -248,10 +249,23 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
                if info and isinstance(info.get('comments'), list):
                    break
            except Exception as e:
-                logger.error(f'Got error when attempting to download comments: {url=}', exc_info=e)
-        else:
+                if 'no longer available' in str(e):
+                    logger.error(f'Giving up on downloading comments because video does not exist: {url=}', exc_info=e)
+                    break
+                if 'Comments are turned off' in str(e):
+                    logger.error(f'Giving up on downloading comments because comments are disabled: {url=}', exc_info=e)
+                    break
+                if 'Private video' in str(e):
+                    logger.error(f'Giving up on downloading comments because video is private: {url=}', exc_info=e)
+                    break
+                if 'Video unavailable' in str(e):
+                    logger.error(f'Giving up on downloading comments because video is unavailable: {url=}', exc_info=e)
+                    break
+                logger.error(f'Got error when attempting to download video comments: {url=}', exc_info=e)
+                await asyncio.sleep(random.randint(1, 5))
+
+        if not info or not isinstance(info.get('comments'), list):
            logger.error(f'Never got comments for video: {url=}')
-            download_manager.add_to_skip_list(url)
            add_video_to_skip_list(url)
            continue

--- a/pi-gen/stage2/04-wrolpi/01-packages
+++ b/pi-gen/stage2/04-wrolpi/01-packages
@ -8,6 +8,7 @@ bzip2
 ca-certificates
 calibre
 cargo
+catdoc
 chromium-browser
 chromium-chromedriver
 cpufrequtils
--- a/requirements.txt
+++ b/requirements.txt
@ -16,6 +16,7 @@ pypdf==3.17.0
 pytest-asyncio==0.21.0
 pytest==7.3.1
 python-dateutil==2.8.2
+python-docx==1.1.0
 python-dotenv==1.0.0
 python-magic~=0.4.27
 pytz==2023.3
--- a/test/example
+++ b/test/example
--- a/test/example
+++ b/test/example
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@ -23,7 +23,7 @@ from itertools import islice, filterfalse, tee
 from multiprocessing import Lock, Manager
 from pathlib import Path
 from types import GeneratorType
-from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set
+from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set, Coroutine
 from urllib.parse import urlparse, urlunsplit

 import aiohttp
@ -1183,13 +1183,24 @@ def chunks_by_stem(it: List[Union[pathlib.Path, str]], size: int) -> Generator[L


@contextlib.contextmanager
-def timer(name):
+def timer(name, level: str = 'debug'):
    """Prints out the time elapsed during the call of some block."""
    before = datetime.now()
+    log_method = getattr(logger_, level)
    try:
        yield
    finally:
-        logger_.warning(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
+        log_method(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
+
+
+def async_timer(coro: Coroutine, name: str = 'async timer', level: str = 'debug') -> callable:
+    """Returns a new coroutine which prints out time elapsed when calling the coroutine."""
+
+    async def _():
+        with timer(name, level):
+            return await coro
+
+    return _()


@contextlib.contextmanager
@ -1270,7 +1281,7 @@ def limit_concurrent(limit: int, throw: bool = False):
    return wrapper


-def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> Union[List[str], str]:
+def truncate_object_bytes(obj: Union[List[str], str, None, Generator], maximum_bytes: int) -> Union[List[str], str]:
    """
    Shorten an object.  This is useful when inserting something into a tsvector.

--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@ -388,6 +388,20 @@ def example_mobi(test_directory) -> pathlib.Path:
    yield destination


+@pytest.fixture
+def example_doc(test_directory) -> pathlib.Path:
+    destination = test_directory / 'example word.doc'
+    shutil.copy(PROJECT_DIR / 'test/example word.doc', destination)
+    yield destination
+
+
+@pytest.fixture
+def example_docx(test_directory) -> pathlib.Path:
+    destination = test_directory / 'example word.docx'
+    shutil.copy(PROJECT_DIR / 'test/example word.docx', destination)
+    yield destination
+
+
@pytest.fixture
 def make_files_structure(test_directory) -> Callable[[Union[List[Union[pathlib.Path, str]], dict]], List[pathlib.Path]]:
    """
--- a/wrolpi/dates.py
+++ b/wrolpi/dates.py
@ -1,7 +1,7 @@
 import logging
 from datetime import datetime, timezone, timedelta
 from enum import Enum
-from typing import Union
+from typing import Union, List

 import dateutil.parser
 import pytz
@ -192,3 +192,44 @@ class TZDateTime(types.TypeDecorator):
            # Assume the DB timestamp is UTC.
            value = value.replace(tzinfo=pytz.utc) if not value.tzinfo else value
        return value
+
+
+def months_selector_to_where(months: List[int], wheres: list, params: dict):
+    """Filter FileGroup results by the month that the file was published.  Returns wheres/params unchanged if months
+    is empty.
+
+    @param months: List of integers representing the index of the month of the year (January is 1).
+    @param wheres: A list of strings which are SQL where filters.
+    @param params: Dict which contains the parameters passed to SQLAlchemy query.
+    """
+    if not months:
+        return wheres, params
+
+    if not set(months).issubset(set(range(1, 13))):
+        raise ValueError('Months must be between 1 and 12')
+
+    wheres.append('EXTRACT (MONTH FROM published_datetime) IN %(months_list)s')
+    params['months_list'] = tuple(months)
+    return wheres, params
+
+
+def date_range_to_where(from_year: int, to_year: int, wheres: list, params: dict):
+    """Filter FileGroup results by the year that the file was published.
+
+    @param from_year: The earliest year the file was published
+    @param to_year: The latest year the file was published
+    @param wheres: A list of strings which are SQL where filters.
+    @param params: Dict which contains the parameters passed to SQLAlchemy query.
+    """
+    if from_year and to_year:
+        wheres.append('EXTRACT (YEAR FROM published_datetime) BETWEEN %(from_year)s AND %(to_year)s')
+        params['from_year'] = from_year
+        params['to_year'] = to_year
+    elif from_year:
+        wheres.append('EXTRACT (YEAR FROM published_datetime) >= %(from_year)s')
+        params['from_year'] = from_year
+    elif to_year:
+        wheres.append('EXTRACT (YEAR FROM published_datetime) <= %(to_year)s')
+        params['to_year'] = to_year
+
+    return wheres, params
--- a/wrolpi/files/api.py
+++ b/wrolpi/files/api.py
@ -100,7 +100,7 @@ async def refresh_progress(request: Request):
@validate(schema.FilesSearchRequest)
 async def post_search_files(_: Request, body: schema.FilesSearchRequest):
    file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
-                                          body.tag_names, body.headline)
+                                          body.tag_names, body.headline, body.months, body.from_year, body.to_year)
    return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))


--- a/wrolpi/files/downloader.py
+++ b/wrolpi/files/downloader.py
@ -55,11 +55,12 @@ class FileDownloader(Downloader, ABC):
            background_task(save_and_tag(output_path, download.settings.get('tag_names')))

            with get_db_session(commit=True) as session:
-                FileGroup.from_paths(session, output_path)
+                fg = FileGroup.from_paths(session, output_path)
+                location = fg.location

            return DownloadResult(
                success=True,
-                location=f'/download/{output_path.relative_to(media_directory)}'
+                location=location,
            )
        except Exception as e:
            logger.error(f'Failed to download {repr(str(download.url))}', exc_info=e)
--- a/wrolpi/files/indexers.py
+++ b/wrolpi/files/indexers.py
@ -1,9 +1,12 @@
 import pathlib
+import subprocess
 from abc import ABC
 from collections import defaultdict
 from typing import List, Type, Tuple
 from zipfile import ZipFile

+import docx
+
 from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE

 try:
@ -135,3 +138,38 @@ class HTMLIndexer(Indexer, ABC):
        title = metadata.title or get_title_from_html(contents)

        return title, a, metadata.description, words
+
+
+@register_indexer('application/msword')
+class DocIndexer(Indexer, ABC):
+    """Extracts words from old Doc files."""
+
+    @classmethod
+    def create_index(cls, path: pathlib.Path):
+        a = cls.get_title(path)
+
+        cmd = ('catdoc', str(path.absolute()))
+        proc = subprocess.run(cmd, capture_output=True)
+        text = proc.stdout.decode()
+        text = truncate_object_bytes(text, FILE_MAX_TEXT_SIZE)
+        words = split_lines_by_length(text)
+
+        return a, None, None, words
+
+
+@register_indexer('application/vnd.openxmlformats-officedocument.wordprocessingml.document')
+class DocxIndexer(Indexer, ABC):
+    """Extracts words from Docx files."""
+
+    @classmethod
+    def create_index(cls, path: pathlib.Path):
+        a = cls.get_title(path)
+
+        doc = docx.Document(str(path))
+        text = ' '.join(truncate_object_bytes(
+            (i.text for i in doc.paragraphs),
+            FILE_MAX_TEXT_SIZE)
+        )
+        words = split_lines_by_length(text)
+
+        return a, None, None, words
--- a/wrolpi/files/lib.py
+++ b/wrolpi/files/lib.py
@ -23,8 +23,9 @@ from wrolpi import flags
 from wrolpi.cmd import which
 from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
    partition, cancelable_wrapper, \
-    get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, get_config
-from wrolpi.dates import now, from_timestamp
+    get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, get_config, \
+    timer
+from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
 from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
 from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
    NoPrimaryFile, InvalidDirectory
@ -540,7 +541,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr

    refreshing_all_files = False

-    with flags.refreshing:
+    with flags.refreshing, timer('refresh_files', 'info'):
        if not paths:
            refresh_logger.warning('Refreshing all files')
            refreshing_all_files = True
@ -684,21 +685,22 @@ async def search_directories_by_name(name: str, excluded: List[str] = None, limi


 def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
-                 tag_names: List[str] = None, headline: bool = False) -> \
+                 tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
+                 from_year: int = None, to_year: int = None) -> \
        Tuple[List[dict], int]:
    """Search the FileGroup table.

    Order the returned Files by their rank if `search_str` is provided.  Return all files if
    `search_str` is empty.

-    Parameters:
-        search_str: Search the ts_vector of the file.  Returns all files if this is empty.
-        limit: Return only this many files.
-        offset: Offset the query.
-        mimetypes: Only return files that match these mimetypes.
-        model: Only return files that match this model.
-        tag_names: A list of tag names.
-        headline: Includes Postgresql headline if True.
+    @param search_str: Search the ts_vector of the file.  Returns all files if this is empty.
+    @param limit: Return only this many files.
+    @param offset: Offset the query.
+    @param mimetypes: Only return files that match these mimetypes.
+    @param model: Only return files that match this model.
+    @param tag_names: A list of tag names.
+    @param headline: Includes Postgresql headline if True.
+    @param months: A list of integers representing the index of the month of the year, starting at 1.
    """
    params = dict(offset=offset, limit=limit, search_str=search_str, url_search_str=f'%{search_str}%')
    wheres = []
@ -721,6 +723,8 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
        wheres.append('model = %(model)s')

    wheres, params = tag_append_sub_select_where(tag_names, wheres, params)
+    wheres, params = months_selector_to_where(months, wheres, params)
+    wheres, params = date_range_to_where(from_year, to_year, wheres, params)

    if search_str and headline:
        headline = ''',
@ -1077,25 +1081,28 @@ def mimetypes_to_sql_wheres(mimetypes: List[str], params: dict):
    return '', params


-async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str]):
+async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str],
+                                       months: List[int] = None, from_year: int = None, to_year: int = None):
    """
    Return FileGroup count of what will be returned if the search is actually performed.
    """
    wheres = []
    joins = list()
-    params = dict(search_str=search_str, tag_names=tag_names, mimetypes=mimetypes, url_search_str=f'%{search_str}%')
+    params = dict(search_str=search_str, tag_names=tag_names, mimetypes=mimetypes)
    group_by = ''
    having = ''

    if search_str:
        # Search by textsearch, and by matching url.
-        wheres.append('(fg.textsearch @@ websearch_to_tsquery(%(search_str)s) OR fg.url ILIKE %(url_search_str)s)')
+        wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')

    wheres, params = tag_append_sub_select_where(tag_names, wheres, params)

    mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params)
    if mimetype_wheres:
        wheres.append(mimetype_wheres)
+    wheres, params = months_selector_to_where(months, wheres, params)
+    wheres, params = date_range_to_where(from_year, to_year, wheres, params)

    joins = "\n".join(joins)
    wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
--- a/wrolpi/files/models.py
+++ b/wrolpi/files/models.py
@ -1,6 +1,7 @@
 import copy
 import pathlib
 import shutil
+import urllib.parse
 from datetime import datetime
 from typing import List, Type, Optional

@ -9,10 +10,10 @@ from sqlalchemy import types
 from sqlalchemy.orm import deferred, relationship, Session
 from sqlalchemy.orm.collections import InstrumentedList

-from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory
+from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
+    get_relative_to_media_directory
 from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
 from wrolpi.db import optional_session
-from wrolpi.downloader import download_manager
 from wrolpi.errors import FileGroupIsTagged
 from wrolpi.files import indexers
 from wrolpi.media_path import MediaPathType
@ -336,6 +337,18 @@ class FileGroup(ModelHelper, Base):
        # Flush the changes to the FileGroup.
        Session.object_session(self).flush([self, ])

+    @property
+    def location(self):
+        """Returns the URL that the FileGroup can be previewed."""
+        parent = str(get_relative_to_media_directory(self.primary_path.parent))
+        preview = str(get_relative_to_media_directory(self.primary_path))
+        if parent == '.':
+            # File is in the top of the media directory, App already shows top directory open.
+            query = urllib.parse.urlencode(dict(preview=str(preview)))
+        else:
+            query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
+        return f'/files?{query}'
+

 class Directory(ModelHelper, Base):
    """A representation of a file directory in the media directory."""
--- a/wrolpi/files/schema.py
+++ b/wrolpi/files/schema.py
@ -26,6 +26,9 @@ class FilesSearchRequest:
    model: Optional[str] = None
    tag_names: List[str] = field(default_factory=list)
    headline: bool = False
+    months: Optional[List[int]] = None
+    from_year: Optional[int] = None
+    to_year: Optional[int] = None


@dataclass
--- a/wrolpi/files/test/test_lib.py
+++ b/wrolpi/files/test/test_lib.py
@ -176,6 +176,20 @@ async def test_refresh_files(test_session, make_files_structure, assert_file_gro
    assert_file_groups([{'primary_path': 'bar.txt', 'indexed': True}])


+@pytest.mark.asyncio
+async def test_file_group_location(test_session, make_files_structure):
+    """FileGroup can return the URL necessary to preview the FileGroup."""
+    make_files_structure({
+        'foo/one.txt': 'one',
+        'two.txt': 'two',
+    })
+    await lib.refresh_files()
+
+    one, two = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
+    assert one.location == '/files?folders=foo&preview=foo%2Fone.txt'
+    assert two.location == '/files?preview=two.txt'
+
+
@pytest.mark.asyncio
 async def test_refresh_bogus_files(test_session, make_files_structure, test_directory, assert_file_groups,
                                   insert_file_group):
@ -945,3 +959,53 @@ async def test_html_index(test_session, test_directory, make_files_structure):
 Some Text
 Span Text
 Outside element.'''
+
+
+@pytest.mark.asyncio
+async def test_doc_indexer(test_session, example_doc):
+    """The contents of a Microsoft doc files can be indexed."""
+    await lib.refresh_files()
+    assert test_session.query(FileGroup).count() == 1
+    doc = test_session.query(FileGroup).one()
+    assert doc.title == 'example word.doc'
+    assert doc.a_text == 'example word doc'
+    assert doc.d_text == 'Example Word Document\n\nSecond page\n'
+
+
+@pytest.mark.asyncio
+async def test_docx_indexer(test_session, example_docx):
+    """The contents of a Microsoft docx files can be indexed."""
+    await lib.refresh_files()
+    assert test_session.query(FileGroup).count() == 1
+    doc = test_session.query(FileGroup).one()
+    assert doc.title == 'example word.docx'
+    assert doc.a_text == 'example word docx'
+    assert doc.d_text == 'Example Word Document Second page'
+
+
+@pytest.mark.asyncio
+async def test_file_search_date_range(test_session, example_pdf, example_doc):
+    """Test searching FileGroups by their published datetime."""
+    await lib.refresh_files()
+    doc, pdf = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
+
+    files, total = lib.search_files('', 10, 0)
+    assert total == 2, 'Should only be 2 files.'
+
+    # PDF was published in December.
+    files, total = lib.search_files('', 10, 0, months=[12, ])
+    assert total == 1, 'Only PDF should be from December 2022'
+    assert files[0]['id'] == pdf.id
+
+    # PDF was published in 2022.
+    files, total = lib.search_files('', 10, 0, from_year=2022, to_year=2022)
+    assert total == 1, 'Only PDF should be from December 2022'
+    assert files[0]['id'] == pdf.id
+
+    # PDF was NOT published in 2023.
+    files, total = lib.search_files('', 10, 0, from_year=2023)
+    assert total == 0, 'No example files are published in 2023'
+
+    files, total = lib.search_files('', 10, 0, to_year=2022)
+    assert total == 1
+    assert files[0]['id'] == pdf.id
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@ -31,7 +31,7 @@ from wrolpi.errors import WROLModeEnabled, APIError, HotspotError, InvalidDownlo
 from wrolpi.events import get_events, Events
 from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
 from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
-    truthy_arg
+    truthy_arg, IS_RPI, IS_RPI4, IS_RPI5
 from wrolpi.version import __version__

 logger = logger.getChild(__name__)
@ -384,6 +384,9 @@ async def get_status(_: Request):
        cpu_info=s.cpu_info,
        disk_bandwidth=s.disk_bandwidth,
        dockerized=DOCKERIZED,
+        is_rpi=IS_RPI,
+        is_rpi4=IS_RPI4,
+        is_rpi5=IS_RPI5,
        downloads=downloads,
        drives=s.drives,
        flags=flags.get_flags(),
@ -515,11 +518,18 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ
    from modules.videos.channel.lib import search_channels_by_name
    from modules.archive.lib import search_domains_by_name

-    file_groups, channels, domains = await asyncio.gather(
-        search_file_suggestion_count(body.search_str, body.tag_names, body.mimetypes),
-        search_channels_by_name(body.search_str),
-        search_domains_by_name(body.search_str),
+    file_coro = search_file_suggestion_count(
+        body.search_str,
+        body.tag_names,
+        body.mimetypes,
+        body.months,
+        body.from_year,
+        body.to_year,
    )
+    channels_coro = search_channels_by_name(body.search_str)
+    domains_coro = search_domains_by_name(body.search_str)
+
+    file_groups, channels, domains = await asyncio.gather(file_coro, channels_coro, domains_coro)

    ret = dict(
        file_groups=file_groups,
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@ -143,3 +143,6 @@ class SearchSuggestionsRequest:
    search_str: Optional[str] = None
    mimetypes: List[str] = field(default_factory=lambda: list())
    tag_names: List[str] = field(default_factory=lambda: list())
+    months: Optional[List[int]] = None
+    from_year: Optional[int] = None
+    to_year: Optional[int] = None
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@ -507,10 +507,10 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
    await assert_results(
        dict(search_str='foo l'),
        [
-         {'directory': 'Fool',
-          'id': 2,
-          'name': 'Fool',
-          'url': 'https://example.com/Fool'}],
+            {'directory': 'Fool',
+             'id': 2,
+             'name': 'Fool',
+             'url': 'https://example.com/Fool'}],
        [],
        0,
    )
@ -555,6 +555,11 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
    # No PDFs.
    await assert_results(dict(mimetypes=['application/pdf']))

+    # Can filter by published datetime.
+    await assert_results(dict(months=[1, ]), expected_file_groups=2)
+    await assert_results(dict(months=[2, ]), expected_file_groups=0)
+    await assert_results(dict(from_year=2000, to_year=2000), expected_file_groups=2)
+

 def test_recursive_errors():
    """Errors are reported recursively."""
--- a/wrolpi/vars.py
+++ b/wrolpi/vars.py
@ -1,5 +1,6 @@
 import multiprocessing
 import os
+import pathlib
 import sys
 from pathlib import Path

@ -80,3 +81,9 @@ VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 50
 DEFAULT_HTTP_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116 Safari/537.36',
 }
+
+RPI_DEVICE_PATH = pathlib.Path('/proc/device-tree/model')
+RPI_DEVICE_MODEL_CONTENTS = RPI_DEVICE_PATH.read_text() if RPI_DEVICE_PATH.is_file() else None
+IS_RPI4 = 'Raspberry Pi 4' in RPI_DEVICE_MODEL_CONTENTS if RPI_DEVICE_MODEL_CONTENTS else False
+IS_RPI5 = 'Raspberry Pi 5' in RPI_DEVICE_MODEL_CONTENTS if RPI_DEVICE_MODEL_CONTENTS else False
+IS_RPI = IS_RPI4 or IS_RPI5