From 00fe799b8f46d8187dcb27ab01b1307dcd23bb15 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 6 Mar 2024 17:41:04 -0700
Subject: [PATCH] User can select from/to years when searching files.  Indexing
 doc and docx files.

---
 .circleci/config.yml                          |  10 +-
 alembic/versions/1c041aeee0fc_doc_indexer.py  |  36 +++
 app/src/App.js                                |  26 +-
 app/src/DashboardPage.js                      |  49 +++-
 app/src/api.js                                |  24 +-
 app/src/components/Common.js                  |  17 +-
 app/src/components/DatesSelector.js           | 230 ++++++++++++++++++
 app/src/components/FilePreview.js             |   2 +-
 app/src/components/Files.js                   |  16 +-
 app/src/components/Search.js                  |  40 ++-
 app/src/components/Videos.js                  |   4 +-
 app/src/contexts/contexts.js                  |   7 +
 app/src/hooks/customHooks.js                  | 165 +++++++++----
 .../config/package-lists/wrolpi.list.chroot   |   1 +
 docker-compose.yml                            |  30 ++-
 docker/api/Dockerfile                         |   2 +-
 main.py                                       |   3 +
 modules/map/lib.py                            |  44 +++-
 modules/map/test/test_lib.py                  |  19 ++
 modules/videos/channel/lib.py                 |   4 +-
 modules/videos/downloader.py                  |   1 -
 modules/videos/video/lib.py                   |  22 +-
 pi-gen/stage2/04-wrolpi/01-packages           |   1 +
 requirements.txt                              |   1 +
 test/example word.doc                         | Bin 0 -> 9216 bytes
 test/example word.docx                        | Bin 0 -> 4322 bytes
 wrolpi/common.py                              |  19 +-
 wrolpi/conftest.py                            |  14 ++
 wrolpi/dates.py                               |  43 +++-
 wrolpi/files/api.py                           |   2 +-
 wrolpi/files/downloader.py                    |   5 +-
 wrolpi/files/indexers.py                      |  38 +++
 wrolpi/files/lib.py                           |  37 +--
 wrolpi/files/models.py                        |  17 +-
 wrolpi/files/schema.py                        |   3 +
 wrolpi/files/test/test_lib.py                 |  64 +++++
 wrolpi/root_api.py                            |  20 +-
 wrolpi/schema.py                              |   3 +
 wrolpi/test/test_root_api.py                  |  13 +-
 wrolpi/vars.py                                |   7 +
 40 files changed, 880 insertions(+), 159 deletions(-)
 create mode 100644 alembic/versions/1c041aeee0fc_doc_indexer.py
 create mode 100644 app/src/components/DatesSelector.js
 create mode 100644 test/example word.doc
 create mode 100644 test/example word.docx

diff --git a/.circleci/config.yml b/.circleci/config.yml
index acb7e620..0d5b1a30 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -73,7 +73,7 @@ jobs:
     steps:
       - checkout
       - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
       - restore_cache:
           key: deps-3.8-{{ checksum "requirements.txt" }}-2
       - run:
@@ -100,7 +100,7 @@ jobs:
     steps:
       - checkout
       - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
       - restore_cache:
           key: deps-3.9-{{ checksum "requirements.txt" }}-2
       - run:
@@ -127,7 +127,7 @@ jobs:
     steps:
       - checkout
       - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
       - restore_cache:
           key: deps-3.10-{{ checksum "requirements.txt" }}-2
       - run:
@@ -154,7 +154,7 @@ jobs:
     steps:
       - checkout
       - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
       - restore_cache:
           key: deps-3.11-{{ checksum "requirements.txt" }}-2
       - run:
@@ -181,7 +181,7 @@ jobs:
     steps:
       - checkout
       - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg
+      - run: sudo apt-get install -y ffmpeg catdoc
       - restore_cache:
           key: deps-3.12-{{ checksum "requirements.txt" }}-2
       - run:
diff --git a/alembic/versions/1c041aeee0fc_doc_indexer.py b/alembic/versions/1c041aeee0fc_doc_indexer.py
new file mode 100644
index 00000000..718d891e
--- /dev/null
+++ b/alembic/versions/1c041aeee0fc_doc_indexer.py
@@ -0,0 +1,36 @@
+"""Doc indexer.
+
+Revision ID: 1c041aeee0fc
+Revises: d96a04eb2785
+Create Date: 2024-03-05 16:54:34.478974
+
+"""
+import os
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = '1c041aeee0fc'
+down_revision = 'd96a04eb2785'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    # Re-index Word documents.
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    session.execute(
+        """
+        UPDATE file_group
+        SET indexed=false
+        WHERE mimetype in
+            ('application/msword','application/vnd.openxmlformats-officedocument.wordprocessingml.document')
+   """)
+
+
+def downgrade():
+    pass
diff --git a/app/src/App.js b/app/src/App.js
index d4b881e0..cd82db13 100644
--- a/app/src/App.js
+++ b/app/src/App.js
@@ -10,7 +10,7 @@ import {MoreRoute} from "./components/Apps";
 import {InventoryRoute} from "./components/Inventory";
 import {ArchiveRoute} from "./components/Archive";
 import {FilesRoute} from "./components/Files";
-import {StatusProvider} from "./hooks/customHooks";
+import {QueryProvider, StatusProvider} from "./hooks/customHooks";
 import {MapRoute} from "./components/Map";
 import {MediaContextProvider, mediaStyles, StatusContext, ThemeContext} from "./contexts/contexts";
 import {Header, ThemeProvider} from "./components/Theme";
@@ -61,17 +61,19 @@ function HelpPage() {
 }
 
 function Root() {
-    return <ThemeProvider>
-        <TagsProvider>
-            <FilePreviewProvider>
-                <header>
-                    <NavBar/>
-                </header>
-                <Outlet/>
-                <Footer/>
-            </FilePreviewProvider>
-        </TagsProvider>
-    </ThemeProvider>
+    return <QueryProvider>
+        <ThemeProvider>
+            <TagsProvider>
+                <FilePreviewProvider>
+                    <header>
+                        <NavBar/>
+                    </header>
+                    <Outlet/>
+                    <Footer/>
+                </FilePreviewProvider>
+            </TagsProvider>
+        </ThemeProvider>
+    </QueryProvider>
 }
 
 const router = createBrowserRouter(createRoutesFromElements(<Route
diff --git a/app/src/DashboardPage.js b/app/src/DashboardPage.js
index 78309cbf..6af9d375 100644
--- a/app/src/DashboardPage.js
+++ b/app/src/DashboardPage.js
@@ -26,7 +26,7 @@ import {SearchView, useSearch, useSearchSuggestions} from "./components/Search";
 import {KiwixRestartMessage, OutdatedZimsMessage} from "./components/Zim";
 import {useWROLMode} from "./hooks/customHooks";
 import {FileSearchFilterButton} from "./components/Files";
-import {FilePreviewContext} from "./components/FilePreview";
+import {dateRangeIsEmpty, DateSelectorButton} from "./components/DatesSelector";
 
 function FlagsMessages({flags}) {
     const {settings, fetchSettings} = React.useContext(SettingsContext);
@@ -170,11 +170,11 @@ export function DashboardPage() {
         loading,
         setSearchStr: setSuggestionSearchStr,
         setSearchTags,
+        months, setMonths,
+        dateRange, setDateRange,
     } = useSearchSuggestions(searchStr, activeTags);
     const {status} = useContext(StatusContext);
 
-    const {setPreviewFile} = React.useContext(FilePreviewContext);
-
     React.useEffect(() => {
         setSuggestionSearchStr(localSearchStr);
     }, [localSearchStr]);
@@ -231,24 +231,53 @@ export function DashboardPage() {
         <Media at='mobile'>
             <Grid>
                 <Grid.Row columns={2}>
-                    <Grid.Column width={13}>
-                        {getSearchResultsInput({size: 'large'})}
+                    <Grid.Column width={12}>
+                        {getSearchResultsInput()}
                     </Grid.Column>
-                    <Grid.Column width={3} textAlign='right'>
-                        <FileSearchFilterButton size='large'/>
+                    <Grid.Column width={1} textAlign='right' style={{padding: 0}}>
+                        <DateSelectorButton onMonthsChange={setMonths} defaultMonthsSelected={months}
+                                            defaultDateRange={dateRange}
+                                            onDateRangeChange={setDateRange}/>
+                    </Grid.Column>
+                    <Grid.Column width={1} textAlign='right'>
+                        <FileSearchFilterButton/>
                     </Grid.Column>
                 </Grid.Row>
             </Grid>
         </Media>
-        <Media greaterThanOrEqual='tablet'>
+        <Media at='tablet'>
             <Grid>
                 <Grid.Row columns={2}>
-                    <Grid.Column mobile={14}>
-                        {getSearchResultsInput({size: 'big'})}
+                    <Grid.Column textAlign='right' width={2}>
+                        <DateSelectorButton onMonthsChange={setMonths} defaultMonthsSelected={months}
+                                            defaultDateRange={dateRange}
+                                            onDateRangeChange={setDateRange}
+                                            buttonProps={{size: 'big'}}/>
                     </Grid.Column>
                     <Grid.Column textAlign='right' width={2}>
                         <FileSearchFilterButton size='big'/>
                     </Grid.Column>
+                    <Grid.Column mobile={12}>
+                        {getSearchResultsInput({size: 'big'})}
+                    </Grid.Column>
+                </Grid.Row>
+            </Grid>
+        </Media>
+        <Media greaterThanOrEqual='computer'>
+            <Grid>
+                <Grid.Row columns={2}>
+                    <Grid.Column textAlign='right' width={1}>
+                        <DateSelectorButton onMonthsChange={setMonths} defaultMonthsSelected={months}
+                                            defaultDateRange={dateRange}
+                                            onDateRangeChange={setDateRange}
+                                            buttonProps={{size: 'big'}}/>
+                    </Grid.Column>
+                    <Grid.Column textAlign='right' width={1}>
+                        <FileSearchFilterButton size='big'/>
+                    </Grid.Column>
+                    <Grid.Column mobile={14}>
+                        {getSearchResultsInput({size: 'big'})}
+                    </Grid.Column>
                 </Grid.Row>
             </Grid>
         </Media>
diff --git a/app/src/api.js b/app/src/api.js
index b407b351..da3a838c 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -1,5 +1,6 @@
 import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API, ZIM_API} from "./components/Common";
 import {toast} from "react-semantic-toasts-2";
+import _ from "lodash";
 
 function timeoutPromise(ms, promise) {
     // Create a timeout wrapper around a promise.  If the timeout is reached, throw an error.  Otherwise, return
@@ -545,7 +546,7 @@ export async function deleteDownload(downloadId) {
     }
 }
 
-export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline) {
+export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear) {
     const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
     if (mimetypes) {
         body['mimetypes'] = mimetypes;
@@ -559,7 +560,16 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta
     if (headline) {
         body['headline'] = true;
     }
-    console.debug('searching files', body);
+    if (!_.isEmpty(months)) {
+        body['months'] = months.map(i => parseInt(i));
+    }
+    if (fromYear) {
+        body['from_year'] = fromYear;
+    }
+    if (toYear) {
+        body['to_year'] = toYear;
+    }
+    console.info('searching files', body);
     const response = await apiPost(`${API_URI}/files/search`, body);
 
     if (response.status === 200) {
@@ -1050,8 +1060,14 @@ export async function fetchDecoded(vinNumber) {
     }
 }
 
-export async function searchSuggestions(search_str, tagNames, mimetypes) {
-    const body = {search_str: search_str, tag_names: tagNames, mimetypes: mimetypes};
+export async function searchSuggestions(search_str, tagNames, mimetypes, months, dateRange) {
+    months = months ? months.map(i => parseInt(i)) : [];
+
+    const body = {search_str, tag_names: tagNames, mimetypes, months};
+    if (dateRange) {
+        body['from_year'] = dateRange[0];
+        body['to_year'] = dateRange[1];
+    }
     const response = await apiPost(`${API_URI}/search_suggestions`, body);
     if (response.ok) {
         const content = await response.json();
diff --git a/app/src/components/Common.js b/app/src/components/Common.js
index f9ac7288..7b5a30d6 100644
--- a/app/src/components/Common.js
+++ b/app/src/components/Common.js
@@ -1657,7 +1657,22 @@ export function getParentDirectory(filePath) {
 export function MultilineText({text, ...props}) {
     return <div {...props}>
         {text.split('\n').map((line, index, array) =>
-            index === array.length - 1 ? line : <p>{line}</p>
+            index === array.length - 1 ? line : <p key={index}>{line}</p>
         )}
     </div>
 }
+
+export const monthNames = [
+    'January',
+    'February',
+    'March',
+    'April',
+    'May',
+    'June',
+    'July',
+    'August',
+    'September',
+    'October',
+    'November',
+    'December',
+];
diff --git a/app/src/components/DatesSelector.js b/app/src/components/DatesSelector.js
new file mode 100644
index 00000000..54835136
--- /dev/null
+++ b/app/src/components/DatesSelector.js
@@ -0,0 +1,230 @@
+import React from "react";
+import {Button, Form, FormField, Modal, ModalActions, ModalContent, ModalHeader, Segment} from "./Theme";
+import {Checkbox, Dropdown} from "semantic-ui-react";
+import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
+import {monthNames} from "./Common";
+import Message from "semantic-ui-react/dist/commonjs/collections/Message";
+
+export const dateRangeIsEmpty = (dateRange) => {
+    return dateRange[0] === null && dateRange[1] === null;
+}
+
+function MonthsForm({monthsSelected, setMonthsSelected}) {
+    monthsSelected = monthsSelected.map(i => parseInt(i));
+
+    const handleWinter = (e) => {
+        if (e) e.preventDefault();
+        setMonthsSelected([12, 1, 2]);
+    }
+
+    const handleSpring = (e) => {
+        if (e) e.preventDefault();
+        setMonthsSelected([3, 4, 5]);
+    }
+
+    const handleSummer = (e) => {
+        if (e) e.preventDefault();
+        setMonthsSelected([6, 7, 8]);
+    }
+
+    const handleFall = (e) => {
+        if (e) e.preventDefault();
+        setMonthsSelected([9, 10, 11]);
+    }
+
+    const monthCheckbox = (label) => {
+        // "January" == 1, etc.
+        const idx = monthNames.indexOf(label) + 1;
+        const isChecked = monthsSelected.indexOf(idx) >= 0;
+        return <Checkbox
+            label={label}
+            checked={isChecked}
+            size='large'
+            onChange={(e, {checked}) => {
+                // Add or remove month index if the respective checkbox is checked.
+                if (checked) {
+                    setMonthsSelected([...monthsSelected, idx]);
+                } else {
+                    setMonthsSelected(monthsSelected.filter(i => i !== idx));
+                }
+            }}
+        />
+    }
+
+    return <Form>
+        <Grid>
+            <Grid.Row columns={3}>
+                <Grid.Column>{monthCheckbox('January')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('February')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('March')}</Grid.Column>
+            </Grid.Row>
+            <Grid.Row columns={3}>
+                <Grid.Column>{monthCheckbox('April')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('May')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('June')}</Grid.Column>
+            </Grid.Row>
+            <Grid.Row columns={3}>
+                <Grid.Column>{monthCheckbox('July')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('August')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('September')}</Grid.Column>
+            </Grid.Row>
+            <Grid.Row columns={3}>
+                <Grid.Column>{monthCheckbox('October')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('November')}</Grid.Column>
+                <Grid.Column>{monthCheckbox('December')}</Grid.Column>
+            </Grid.Row>
+            <Grid.Row columns={4}>
+                <Grid.Column><Button onClick={handleWinter} color='blue' size='small'>Winter</Button></Grid.Column>
+                <Grid.Column><Button onClick={handleSpring} color='green' size='small'>Spring</Button></Grid.Column>
+                <Grid.Column><Button onClick={handleSummer} color='yellow' size='small'>Summer</Button></Grid.Column>
+                <Grid.Column><Button onClick={handleFall} color='red' size='small'>Fall</Button></Grid.Column>
+            </Grid.Row>
+        </Grid>
+    </Form>
+}
+
+function DateRangeForm({dateRange, setDateRange}) {
+    const currentYear = (new Date()).getFullYear();
+    const [error, setError] = React.useState('');
+
+    let yearRange = [];
+    for (let i = 1970; i <= currentYear; i++) {
+        yearRange = [...yearRange, {
+            key: i,
+            text: i,
+            value: i,
+        }];
+    }
+
+    const handleFromYear = (e, {value}) => {
+        const toYear = dateRange[1] || currentYear;
+        setDateRange([value, toYear]);
+    }
+
+    const handleToYear = (e, {value}) => {
+        setDateRange([dateRange[0], value]);
+    }
+
+    React.useEffect(() => {
+        if (dateRange[0] > dateRange[1]) {
+            setError('From Year must be greater than To Year');
+        } else {
+            setError('');
+        }
+    }, [dateRange])
+
+    const errorMessage = error && <Message error content={error}/>;
+
+    return <React.Fragment>
+        <Form>
+            <Grid>
+                <Grid.Row columns={2}>
+                    <Grid.Column>
+                        <FormField>
+                            <label>From Year</label>
+                            <Dropdown search scrolling fluid
+                                      options={yearRange}
+                                      value={dateRange[0]}
+                                      onChange={handleFromYear}
+                                      error={!!error}
+                            />
+                        </FormField>
+                    </Grid.Column>
+                    <Grid.Column>
+                        <FormField>
+                            <label>To Year</label>
+                            <Dropdown search scrolling fluid
+                                      options={yearRange.reverse()}
+                                      value={dateRange[1]}
+                                      onChange={handleToYear}
+                                      error={!!error}
+                            />
+                        </FormField>
+                    </Grid.Column>
+                </Grid.Row>
+            </Grid>
+        </Form>
+        {errorMessage}
+    </React.Fragment>
+}
+
+export function DateSelectorButton({
+                                       onMonthsChange,
+                                       onDateRangeChange,
+                                       defaultMonthsSelected,
+                                       defaultDateRange,
+                                       buttonProps
+                                   }) {
+    const emptyDateRange = [null, null];
+
+    const [open, setOpen] = React.useState(false);
+    const [monthsSelected, setMonthsSelected] = React.useState(defaultMonthsSelected || []);
+    const [dateRange, setDateRange] = React.useState(defaultDateRange || emptyDateRange);
+    const [color, setColor] = React.useState('grey');
+
+    React.useEffect(() => {
+        if (monthsSelected.length > 0 || (!dateRangeIsEmpty(dateRange))) {
+            setColor('violet');
+        } else {
+            setColor('grey');
+        }
+    }, [JSON.stringify(dateRange), JSON.stringify(monthsSelected)]);
+
+    const handleOpen = (e) => {
+        if (e) e.preventDefault();
+        setOpen(true);
+    }
+
+    const handleClose = (e) => {
+        if (e) e.preventDefault();
+        setOpen(false);
+        // Only submit selection when user has closed selector.
+        if (onDateRangeChange && dateRange[0] <= dateRange[1]) {
+            onDateRangeChange(dateRange);
+        } else if (onDateRangeChange && dateRangeIsEmpty(dateRange)) {
+            //Clear the date range.
+            onDateRangeChange(dateRange);
+        }
+        if (onMonthsChange) {
+            onMonthsChange(monthsSelected);
+        }
+    }
+
+    const localOnSetDateRange = (newDateRange) => {
+        setDateRange(newDateRange);
+    }
+
+    const handleClear = (e) => {
+        if (e) e.preventDefault();
+        setDateRange(emptyDateRange);
+        setMonthsSelected([]);
+    }
+
+    return <React.Fragment>
+        <Button
+            icon='calendar alternate'
+            onClick={handleOpen}
+            color={color}
+            {...buttonProps}
+        />
+        <Modal closeIcon
+               open={open}
+               onClose={handleClose}
+               size='tiny'
+        >
+            <ModalHeader>Filter by Date</ModalHeader>
+            <ModalContent>
+                <Segment>
+                    <DateRangeForm dateRange={dateRange} setDateRange={localOnSetDateRange}/>
+                </Segment>
+                <Segment>
+                    <MonthsForm monthsSelected={monthsSelected} setMonthsSelected={setMonthsSelected}/>
+                </Segment>
+            </ModalContent>
+            <ModalActions>
+                <Button onClick={handleClear} secondary>Clear</Button>
+                <Button onClick={handleClose}>Close</Button>
+            </ModalActions>
+        </Modal>
+    </React.Fragment>
+}
diff --git a/app/src/components/FilePreview.js b/app/src/components/FilePreview.js
index 5b8b8695..b1dcb6ea 100644
--- a/app/src/components/FilePreview.js
+++ b/app/src/components/FilePreview.js
@@ -194,7 +194,7 @@ export function FilePreviewProvider({children}) {
         if (previewQuery && !previewFile) {
             initPreviewFile();
         }
-    }, []);
+    }, [previewQuery]);
 
     const localFetchFile = async () => {
         // Get the file again with its Tags.
diff --git a/app/src/components/Files.js b/app/src/components/Files.js
index c0a4054b..eca3fba4 100644
--- a/app/src/components/Files.js
+++ b/app/src/components/Files.js
@@ -32,9 +32,9 @@ import {
     useTitle
 } from "./Common";
 import {
+    useAllQuery,
     useFilesProgressInterval,
     usePages,
-    useQuery,
     useSearchFiles,
     useSearchFilter,
     useSearchView,
@@ -149,7 +149,7 @@ function FileCard({file}) {
                 </ExternalCardLink>
             </CardHeader>
             {author && <b {...s}>{author}</b>}
-            <p>{isoDatetimeToString(file.modified)}</p>
+            <p>{isoDatetimeToString(file.published_datetime) || isoDatetimeToString(file.modified)}</p>
             <p>{size}</p>
         </CardContent>
     </Card>
@@ -403,6 +403,9 @@ export function SearchFilter({filters = [], modalHeader, size = 'medium'}) {
         </Form.Field>
     );
 
+    // Use violet color when filter has been applied.
+    const buttonColor = filter ? 'violet' : 'grey';
+
     if (filters && filters.length > 0) {
         return <>
             <Modal open={open} onOpen={() => handleOpen()} onClose={() => setOpen(false)} closeIcon>
@@ -420,7 +423,7 @@ export function SearchFilter({filters = [], modalHeader, size = 'medium'}) {
             <Button
                 icon='filter'
                 onClick={handleOpen}
-                primary={!!filter}
+                color={buttonColor}
                 size={size}
             />
         </>
@@ -430,12 +433,11 @@ export function SearchFilter({filters = [], modalHeader, size = 'medium'}) {
 }
 
 export function TagsQuerySelector({onChange}) {
-    // Creates  dropdown that the User can use to manipulate the tag query.
-    const {searchParams, updateQuery} = useQuery();
-    const activeTags = searchParams.getAll('tag');
+    // Creates modal that the User can use to manipulate the tag query.
+    const [activeTags, setActiveTags] = useAllQuery('tag');
 
     const localOnChange = (tagNames) => {
-        updateQuery({'tag': tagNames});
+        setActiveTags(tagNames);
         if (onChange) {
             onChange(tagNames);
         }
diff --git a/app/src/components/Search.js b/app/src/components/Search.js
index 717b6b95..a201965a 100644
--- a/app/src/components/Search.js
+++ b/app/src/components/Search.js
@@ -1,7 +1,7 @@
 import React, {useState} from "react";
-import {createSearchParams, Route, Routes, useNavigate} from "react-router-dom";
+import {Route, Routes, useNavigate} from "react-router-dom";
 import {FilesSearchView} from "./Files";
-import {useLatestRequest, usePages, useQuery, useSearchFilter} from "../hooks/customHooks";
+import {useLatestRequest, usePages, useSearchDateRange, useSearchFilter, useSearchMonths} from "../hooks/customHooks";
 import {ZimSearchView} from "./Zim";
 import {searchEstimateZims, searchSuggestions} from "../api";
 import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common";
@@ -9,6 +9,7 @@ import _ from "lodash";
 import {TagsContext} from "../Tags";
 import {Button as SButton, Header as SHeader, Label} from "semantic-ui-react";
 import {Modal, ModalContent} from "./Theme";
+import {QueryContext} from "../contexts/contexts";
 
 const SUGGESTED_APPS = [
     {location: '/admin', title: 'Downloads', description: 'View and control your downloads'},
@@ -37,7 +38,9 @@ const SUGGESTED_APPS = [
 export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false, model) => {
     const navigate = useNavigate();
 
-    const {searchParams, updateQuery} = useQuery();
+    const {months, setMonths} = useSearchMonths();
+    const {dateRange, setDateRange} = useSearchDateRange();
+    const {searchParams, updateQuery, getLocationStr} = React.useContext(QueryContext);
     // `searchStr` means actually fetch the files/zims.
     const searchStr = searchParams.get('q');
     // User can search only by Tags, `searchStr` not required.
@@ -53,11 +56,8 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
         if (filter) {
             searchQuery['filter'] = filter;
         }
-        navigate({
-            pathname: '/search',
-            // Start new search at offset 0.
-            search: createSearchParams(searchQuery).toString(),
-        });
+        const location = getLocationStr(searchQuery, '/search');
+        navigate(location);
     }
 
     const clearSearch = () => {
@@ -90,6 +90,8 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
         setSearchStr,
         clearSearch,
         setTags,
+        months, setMonths,
+        dateRange, setDateRange,
     }
 }
 
@@ -101,6 +103,8 @@ export function useSuggestions(searchStr, tagNames, filter) {
         zimsEstimates: [],
     }
     const [suggestions, setSuggestions] = React.useState(defaultSuggestions);
+    const {months} = useSearchMonths();
+    const {dateRange} = useSearchDateRange();
     // fileGroups/channels/domains.
     const {data, sendRequest, loading} = useLatestRequest(500);
     // Zims are slow, so they are separate.
@@ -109,10 +113,17 @@ export function useSuggestions(searchStr, tagNames, filter) {
     React.useEffect(() => {
         if ((searchStr && searchStr.length > 0) || (tagNames && tagNames.length > 0)) {
             const mimetypes = filterToMimetypes(filter);
-            sendRequest(async () => await searchSuggestions(searchStr, tagNames, mimetypes));
+            sendRequest(async () => await searchSuggestions(searchStr, tagNames, mimetypes, months, dateRange));
             sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
         }
-    }, [searchStr, sendRequest, sendZimRequest, JSON.stringify(tagNames)]);
+    }, [
+        searchStr,
+        sendRequest,
+        sendZimRequest,
+        JSON.stringify(tagNames),
+        JSON.stringify(months),
+        JSON.stringify(dateRange),
+    ]);
 
     React.useEffect(() => {
         if (!_.isEmpty(data)) {
@@ -141,8 +152,10 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames) {
     const [searchStr, setSearchStr] = React.useState(defaultSearchStr || '');
     const [searchTags, setSearchTags] = React.useState(defaultTagNames);
     const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
+    const {months, setMonths} = useSearchMonths();
+    const {dateRange, setDateRange} = useSearchDateRange();
     const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter);
-    const {getLocationStr} = useQuery();
+    const {getLocationStr} = React.useContext(QueryContext);
 
     // The results that will be displayed by <Search>.
     const [suggestionsResults, setSuggestionsResults] = useState({});
@@ -275,12 +288,13 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames) {
         suggestions,
         suggestionsResults,
         suggestionsSums,
-        searchStr,
-        setSearchStr,
+        searchStr, setSearchStr,
         setSearchTags,
+        months, setMonths,
         handleResultSelect,
         resultRenderer,
         loading,
+        dateRange, setDateRange,
     }
 }
 
diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js
index 387e496d..0d9fa4ea 100644
--- a/app/src/components/Videos.js
+++ b/app/src/components/Videos.js
@@ -52,7 +52,7 @@ import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
 import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
 import {Button, Card, Header, Loader, Placeholder, Segment, Statistic, StatisticGroup} from "./Theme";
 import {deleteVideos} from "../api";
-import {Media, ThemeContext} from "../contexts/contexts";
+import {Media, QueryContext, ThemeContext} from "../contexts/contexts";
 import _ from "lodash";
 
 export function VideoWrapper() {
@@ -69,7 +69,7 @@ export function VideoWrapper() {
 function VideosPage() {
 
     const {channelId} = useParams();
-    const {searchParams} = useQuery();
+    const {searchParams} = React.useContext(QueryContext);
     const [selectedVideos, setSelectedVideos] = useState([]);
 
     let searchOrder = defaultVideoOrder;
diff --git a/app/src/contexts/contexts.js b/app/src/contexts/contexts.js
index e4a734c1..79c5b463 100644
--- a/app/src/contexts/contexts.js
+++ b/app/src/contexts/contexts.js
@@ -13,6 +13,13 @@ export const SettingsContext = React.createContext({
     fetchSettings: null,
 });
 
+export const QueryContext = React.createContext({
+    searchParams: null,
+    setSearchParams: null,
+    updateQuery: null,
+    getLocationStr: null,
+})
+
 export const AppMedia = createMedia({
     breakpoints: {
         mobile: 0, tablet: 700, computer: 1024,
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js
index 7e15057f..9fce1b85 100644
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@@ -15,7 +15,6 @@ import {
     getSettings,
     getStatistics,
     getStatus,
-    getThrottleStatus,
     getVideo,
     getVideosStatistics,
     searchArchives,
@@ -28,7 +27,7 @@ import {
 } from "../api";
 import {createSearchParams, useLocation, useSearchParams} from "react-router-dom";
 import {enumerate, filterToMimetypes, humanFileSize, secondsToFullDuration} from "../components/Common";
-import {SettingsContext, StatusContext} from "../contexts/contexts";
+import {QueryContext, SettingsContext, StatusContext} from "../contexts/contexts";
 import {toast} from "react-semantic-toasts-2";
 import {useSearch} from "../components/Search";
 import _ from "lodash";
@@ -103,63 +102,95 @@ export const useLatestRequest = (delay = 300, defaultLoading = false) => {
     return {data, sendRequest, loading};
 };
 
+const getSearchParamCopy = (searchParams) => {
+    let copy = {};
+    if (searchParams) {
+        Array.from(searchParams.keys()).forEach(key => {
+            const value = searchParams.getAll(key)
+            copy[key] = value.length === 1 ? value[0] : value;
+        })
+    }
+    copy = removeEmptyValues(copy);
+    return copy;
+}
 
-export const useQuery = () => {
+const removeEmptyValues = (obj) => {
+    Object.entries(obj).forEach(([k, v]) => {
+        // Clear any empty values.
+        if (v === null || v === undefined || (Array.isArray(v) && v.length === 0)) {
+            delete obj[k];
+        }
+    });
+    return obj
+}
+
+
+export const useQuery = (defaultParams) => {
     const location = useLocation();
     const [searchParams, setSearchParams] = useSearchParams();
+    const [state, setState] = React.useState(defaultParams || getSearchParamCopy(searchParams));
 
-    const setQuery = (obj, replace = true) => {
-        console.debug(`setQuery`, obj);
-        setSearchParams(createSearchParams(obj), {replace: replace});
-    }
-
-    const getQuery = (obj) => {
-        const newQuery = {};
-        for (const entry of searchParams.entries()) {
-            newQuery[entry[0]] = entry[1];
+    React.useEffect(() => {
+        // Copy the state into the URL.
+        setSearchParams(state);
+        if (!_.isEmpty(state)) {
+            console.debug('Changing URL params', state);
         }
-        Object.entries(obj).forEach(([key, value]) => {
-            if (value === undefined || value === null || value === '') {
-                delete newQuery[key];
-            } else {
-                newQuery[key] = value;
-            }
-        })
-        return newQuery;
-    }
+    }, [state]);
 
-    const updateQuery = (obj) => {
-        const newQuery = getQuery(obj);
-        setQuery(newQuery);
-    }
+    React.useEffect(() => {
+        // Keep URL and state in sync.
+        setState(getSearchParamCopy(searchParams));
+    }, [searchParams]);
+
+    const updateQuery = (newParams, replace = false) => {
+        // Update the old state with the new values.
+        setState(oldState => {
+            if (replace) {
+                // Ignore oldState because all values are going to be replaced.
+                return removeEmptyValues(newParams);
+            } else {
+                return removeEmptyValues({...oldState, ...newParams});
+            }
+        });
+    };
 
     const getLocationStr = (newSearchParams, pathname) => {
         // Get the current location, but with new params appended.
-        const newQuery = createSearchParams(getQuery(newSearchParams));
+        newSearchParams = {...state, ...newSearchParams};
+        const newQuery = createSearchParams(newSearchParams);
         return `${pathname || location.pathname}?${newQuery.toString()}`
     }
 
-    return {searchParams, setSearchParams, setQuery, updateQuery, getLocationStr}
+    return {searchParams, updateQuery, getLocationStr}
+}
+
+
+export const QueryProvider = (props) => {
+    const value = useQuery();
+
+    return <QueryContext.Provider value={value}>
+        {props.children}
+    </QueryContext.Provider>
 }
 
 export const useOneQuery = (name) => {
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
     const value = searchParams.get(name);
 
-    const setValue = (newValue) => {
-        console.debug(`useOneQuery setValue=${newValue}`);
-        updateQuery({[name]: newValue});
+    const setValue = (newValue, replace = false) => {
+        updateQuery({[name]: newValue}, replace);
     }
 
     return [value, setValue]
 }
 
 export const useAllQuery = (name) => {
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
     const value = searchParams.getAll(name);
 
-    const setValue = (newValue) => {
-        updateQuery({[name]: newValue});
+    const setValue = (newValue, replace = false) => {
+        updateQuery({[name]: newValue}, replace);
     }
 
     return [value, setValue]
@@ -212,7 +243,7 @@ export const useArchive = (archiveId) => {
 }
 
 export const usePages = (defaultLimit = 24, totalPages = 0) => {
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
     const offset = searchParams.get('o') || 0;
     const limit = parseInt(searchParams.get('l') || defaultLimit || 24);
     const [activePage, setActivePage] = useState(calculatePage(offset, limit));
@@ -232,7 +263,7 @@ export const usePages = (defaultLimit = 24, totalPages = 0) => {
 
     const setTotal = (total) => {
         const newTotalPages = calculateTotalPages(total, limit);
-        console.log('newTotalPages', newTotalPages);
+        console.debug('newTotalPages', newTotalPages);
         setTotalPages(newTotalPages);
     }
 
@@ -245,7 +276,7 @@ export const usePages = (defaultLimit = 24, totalPages = 0) => {
 export const useSearchArchives = (defaultLimit) => {
     const {domain} = useSearchDomain();
     const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
     const searchStr = searchParams.get('q') || '';
     const order = searchParams.get('order');
     const activeTags = searchParams.getAll('tag');
@@ -304,7 +335,7 @@ export const useSearchArchives = (defaultLimit) => {
 }
 
 export const useSearchVideos = (defaultLimit, channelId, order_by) => {
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
     const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
     const searchStr = searchParams.get('q') || '';
     const order = searchParams.get('order') || order_by;
@@ -473,7 +504,9 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
         searchStr,
         filter,
         model: model_,
-        setSearchStr
+        setSearchStr,
+        months,
+        dateRange,
     } = useSearch(defaultLimit, emptySearch, model);
     const {view} = useSearchView();
 
@@ -486,9 +519,16 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
         }
         const mimetypes = filterToMimetypes(filter);
         setSearchFiles(null);
+        let fromYear;
+        let toYear;
+        if (dateRange) {
+            fromYear = dateRange[0];
+            toYear = dateRange[1];
+        }
         try {
             let [file_groups, total] = await filesSearch(
-                pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline);
+                pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline,
+                months, fromYear, toYear);
             setSearchFiles(file_groups);
             pages.setTotal(total);
         } catch (e) {
@@ -514,7 +554,17 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
         }
         // Handle when this is unmounted.
         return () => debouncedLocalSearchFiles.cancel();
-    }, [searchStr, pages.effect, filter, model, model_, JSON.stringify(activeTags), headline]);
+    }, [
+        searchStr,
+        pages.effect,
+        filter,
+        model,
+        model_,
+        JSON.stringify(activeTags),
+        headline,
+        JSON.stringify(months),
+        JSON.stringify(dateRange),
+    ]);
 
     return {
         searchFiles,
@@ -933,6 +983,33 @@ export const useSearchOrder = () => {
     return {sort, setSort}
 }
 
+export const useSearchMonths = () => {
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
+    const months = searchParams.getAll('month');
+
+    const setMonths = (newMonths) => {
+        // Set new months, go back to first page.
+        updateQuery({'month': newMonths, 'o': 0});
+    }
+
+    return {months, setMonths}
+}
+
+export const useSearchDateRange = () => {
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
+    let fromDate = searchParams.get('fromDate');
+    let toDate = searchParams.get('toDate');
+    fromDate = fromDate ? parseInt(fromDate) : null;
+    toDate = toDate ? parseInt(toDate) : null;
+
+    const setDateRange = ([newFromDate, newToDate]) => {
+        // Set new date range, go back to first page.
+        updateQuery({fromDate: newFromDate, toDate: newToDate, 'o': 0});
+    }
+
+    return {dateRange: [fromDate, toDate], setDateRange}
+}
+
 export const useUploadFile = () => {
     const [files, setFiles] = useState([]);
     const [progresses, setProgresses] = useState({});
@@ -983,7 +1060,7 @@ export const useUploadFile = () => {
                     handleProgress(file.name, chunkNum, totalChunks, 'pending', file.type);
                     const expectedChunk = data['expected_chunk'];
                     if (xhr.status === 416) {
-                        console.log(`Server requested a different chunk ${chunkNum}`);
+                        console.warn(`Server requested a different chunk ${chunkNum}`);
                         await uploadChunk(file, expectedChunk, chunkSize, totalChunks, tries + 1, maxTries);
                     } else {
                         console.debug(`Uploading of chunk ${chunkNum} succeeded, got request for chunk ${chunkNum}`);
@@ -992,7 +1069,7 @@ export const useUploadFile = () => {
                     }
                 } else if (xhr.status === 201) {
                     handleProgress(file.name, totalChunks, totalChunks, 'complete', file.type);
-                    console.log(`Uploading of ${file.path} completed.`);
+                    console.info(`Uploading of ${file.path} completed.`);
                 } else if (xhr.status === 400) {
                     handleProgress(file.name, totalChunks, totalChunks, 'conflicting', file.type);
                     const data = JSON.parse(xhr.responseText);
@@ -1046,7 +1123,7 @@ export const useUploadFile = () => {
 
 export const useSearchZims = (defaultLimit) => {
     const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
-    const {searchParams, updateQuery} = useQuery();
+    const {searchParams, updateQuery} = React.useContext(QueryContext);
     const searchStr = searchParams.get('q') || '';
 
     const [zims, setZims] = useState(null);
diff --git a/debian-live-config/config/package-lists/wrolpi.list.chroot b/debian-live-config/config/package-lists/wrolpi.list.chroot
index d016cad5..67e68c28 100644
--- a/debian-live-config/config/package-lists/wrolpi.list.chroot
+++ b/debian-live-config/config/package-lists/wrolpi.list.chroot
@@ -8,6 +8,7 @@ bzip2
 ca-certificates
 calibre
 cargo
+catdoc
 chromium
 cpufrequtils
 curl
diff --git a/docker-compose.yml b/docker-compose.yml
index 63d1f05c..d9a80792 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,8 +1,7 @@
 # Development containers, these should only be used if you are developing.
 version: '3'
 services:
-  db:
-    # https://github.com/docker-library/docs/tree/master/postgres
+  db: # https://github.com/docker-library/docs/tree/master/postgres
     image: postgres:12
     environment:
       - POSTGRES_DB=wrolpi
@@ -11,6 +10,7 @@ services:
       - "5432:5432"
     healthcheck:
       test: [ 'CMD-SHELL', 'pg_isready -U postgres' ]
+      interval: 10s
 
   api:
     depends_on:
@@ -30,10 +30,11 @@ services:
       - ${REACT_APP_API-127.0.0.1:8081}:8081
     command: '-vv api --host 0.0.0.0'
     user: '${UID-1000}:${GID-1000}'
-  #    healthcheck:
-  #      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]
+      interval: 2m
 
-  archive:
+  archive: # Python/Sanic wrapper around Singlefile and Readability.
     build:
       context: .
       dockerfile: docker/archive/Dockerfile
@@ -43,8 +44,9 @@ services:
       - '8083:8080'
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8080/' ]
+      interval: 2m
 
-  app:
+  app: # React App
     depends_on:
       - api
     build:
@@ -59,13 +61,13 @@ services:
     stdin_open: true
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:3000' ]
+      interval: 1m
     user: '${UID-1000}:${GID-1000}'
     environment:
       # Force React to use nginx port to connect to it's websocket.
       - WDS_SOCKET_PORT=0
 
-  map:
-    # https://github.com/Overv/openstreetmap-tile-server
+  map: # https://github.com/Overv/openstreetmap-tile-server
     image: overv/openstreetmap-tile-server:v1.6.0
     volumes:
       - 'openstreetmap-data:/var/lib/postgresql/12/main'
@@ -77,6 +79,9 @@ services:
       - '8084:80'
     command: 'run'
     shm_size: 1g  # Increase shared size to render large map tiles
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
+      interval: 1m
 
   web:
     depends_on:
@@ -91,8 +96,9 @@ services:
       - './test:/opt/media'
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
+      interval: 1m
 
-  zim:
+  zim: # kiwix-serve
     build:
       context: .
       dockerfile: docker/zim/Dockerfile
@@ -101,6 +107,9 @@ services:
       - './docker/zim/entrypoint.sh:/entrypoint.sh'
     ports:
       - '8085:8085'
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8085' ]
+      interval: 1m
 
   help: # The WROLPi Help documentation.
     build:
@@ -110,6 +119,9 @@ services:
       - './docker/help/wrolpi-help:/opt/wrolpi-help'
     ports:
       - '8086:8086'
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8086' ]
+      interval: 1m
 
 volumes:
   openstreetmap-data:
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index fca52006..6f871ac5 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -5,7 +5,7 @@ WORKDIR /opt/wrolpi
 # Update dependencies for the services install.
 RUN apt update
 # ffmpeg for Videos
-RUN apt-get install -y ffmpeg
+RUN apt-get install -y ffmpeg catdoc
 RUN ffmpeg -version
 
 # Install WROLPi
diff --git a/main.py b/main.py
index 3f3d2df6..aa0cbc01 100755
--- a/main.py
+++ b/main.py
@@ -209,6 +209,9 @@ async def startup(app: Sanic):
             # We can't search for Videos missing comments until the refresh has completed.
             pass
 
+        # Wait for download manager to startup.
+        await asyncio.sleep(5)
+
         while True:
             # Fetch comments for videos every hour.
             if download_manager.disabled.is_set() or download_manager.stopped.is_set():
diff --git a/modules/map/lib.py b/modules/map/lib.py
index fc8f816f..a9541fae 100644
--- a/modules/map/lib.py
+++ b/modules/map/lib.py
@@ -13,7 +13,7 @@ from wrolpi.common import get_media_directory, walk, logger
 from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp
 from wrolpi.db import optional_session, get_db_session
 from wrolpi.events import Events
-from wrolpi.vars import PROJECT_DIR
+from wrolpi.vars import PROJECT_DIR, IS_RPI5
 
 logger = logger.getChild(__name__)
 
@@ -224,16 +224,42 @@ def get_import_status(session: Session = None) -> List[MapFile]:
     return map_paths
 
 
-# Bps calculated using many tests on a well-cooled RPi4.
+# Calculated using many tests on a well-cooled RPi4 (4GB).
 RPI4_PBF_BYTES_PER_SECOND = 61879
+RPI4_A = 7.17509261732342e-14
+RPI4_B = 6.6590760410412e-5
+RPI4_C = 10283
+# Calculated using many tests on a well-cooled RPi5 (8GB).
+RPI5_PBF_BYTES_PER_SECOND = 136003
+RPI5_A = 1.6681382703586e-15
+RPI5_B = 2.35907824676145e-6
+RPI5_C = 53.727
+
+
+def seconds_to_import_rpi4(size_in_bytes: int) -> int:
+    if size_in_bytes > 1_000_000_000:
+        # Use exponential curve for large files.
+        a = RPI4_A * size_in_bytes ** 2
+        b = RPI4_B * size_in_bytes
+        return int(a - b + RPI4_C)
+    # Use simpler equation for small files.
+    return max(int(size_in_bytes // RPI4_PBF_BYTES_PER_SECOND), 0)
+
+
+def seconds_to_import_rpi5(size_in_bytes: int) -> int:
+    if size_in_bytes > 5_000_000_000:
+        # Use exponential curve for large files.
+        a = RPI5_A * size_in_bytes ** 2
+        b = RPI5_B * size_in_bytes
+        return int(a - b + RPI5_C)
+    # Use simpler equation for small files.
+    return max(int(size_in_bytes // RPI5_PBF_BYTES_PER_SECOND), 0)
 
 
 def seconds_to_import(size_in_bytes: int) -> int:
     """Attempt to predict how long it will take an RPi4 to import a given PBF file."""
-    if size_in_bytes > 1_000_000_000:
-        # Use exponential curve for large files.  These magic numbers are from testing many imports on an RPi 4.
-        a = 7.17509261732342e-14 * size_in_bytes ** 2
-        b = 6.6590760410412e-5 * size_in_bytes
-        c = 10283
-        return int(a - b + c)
-    return max(int(size_in_bytes // RPI4_PBF_BYTES_PER_SECOND), 0)
+    if IS_RPI5:
+        return seconds_to_import_rpi5(size_in_bytes)
+
+    # Default to RPi4, because it's the most conservative estimate.
+    return seconds_to_import_rpi4(size_in_bytes)
diff --git a/modules/map/test/test_lib.py b/modules/map/test/test_lib.py
index 58d3d217..7069b404 100644
--- a/modules/map/test/test_lib.py
+++ b/modules/map/test/test_lib.py
@@ -1,3 +1,4 @@
+import unittest.mock
 from unittest import mock
 
 import pytest
@@ -82,3 +83,21 @@ async def test_run_import_command(test_directory, mock_create_subprocess_shell):
 ])
 def test_seconds_to_import_rpi4(size, expected):
     assert lib.seconds_to_import(size) == expected
+
+
+@pytest.mark.parametrize('size,expected', [
+    (0, 0),
+    (-1, 0),
+    (17737381, 130),
+    (63434267, 466),
+    (87745484, 645),
+    (116318111, 855),
+    (136372996, 1002),
+    (1936075318, 14235),
+    (2676094489, 19676),
+    (3392375001, 24943),
+    (11346305075, 188040),
+])
+def test_seconds_to_import_rpi5(size, expected):
+    with unittest.mock.patch('modules.map.lib.IS_RPI5', True):
+        assert lib.seconds_to_import(size) == expected
diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index 9170ae52..e14815e5 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -207,8 +207,8 @@ async def search_channels_by_name(name: str, limit: int = 5, session: Session =
     name_no_spaces = ''.join(name.split(' '))
     stmt = session.query(Channel) \
         .filter(or_(
-        Channel.name.ilike(f'%{name}%'),
-        Channel.name.ilike(f'%{name_no_spaces}%'),
+            Channel.name.ilike(f'%{name}%'),
+            Channel.name.ilike(f'%{name_no_spaces}%'),
     )) \
         .order_by(asc(Channel.name)) \
         .limit(limit)
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 6286e050..f2437a44 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -77,7 +77,6 @@ class ChannelDownloader(Downloader, ABC):
     @staticmethod
     def is_a_playlist(info: dict):
         # A playlist may have an id different from its channel.
-        logger.debug(f'is_a_playlist {info["id"]=} {info.get("channel_id")=}')
         return info['id'] != info.get('channel_id')
 
     async def do_download(self, download: Download) -> DownloadResult:
diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py
index 63be8cbb..9ea3a059 100644
--- a/modules/videos/video/lib.py
+++ b/modules/videos/video/lib.py
@@ -1,5 +1,4 @@
 import asyncio
-import json
 import random
 from datetime import timedelta
 from typing import Tuple, Optional, List, Dict
@@ -218,6 +217,7 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
         with get_db_session(commit=True) as session_:
             for video_ in session_.query(Video).join(FileGroup).filter_by(url=video_url).all():
                 video_.comments_failed = True
+        download_manager.add_to_skip_list(video_url)
 
     for url in video_urls:
         # Sleep to catch cancel.
@@ -240,6 +240,7 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
                 continue
 
         tries = 5
+        info = None
         while tries > 0:
             try:
                 tries -= 1
@@ -248,10 +249,23 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
                 if info and isinstance(info.get('comments'), list):
                     break
             except Exception as e:
-                logger.error(f'Got error when attempting to download comments: {url=}', exc_info=e)
-        else:
+                if 'no longer available' in str(e):
+                    logger.error(f'Giving up on downloading comments because video does not exist: {url=}', exc_info=e)
+                    break
+                if 'Comments are turned off' in str(e):
+                    logger.error(f'Giving up on downloading comments because comments are disabled: {url=}', exc_info=e)
+                    break
+                if 'Private video' in str(e):
+                    logger.error(f'Giving up on downloading comments because video is private: {url=}', exc_info=e)
+                    break
+                if 'Video unavailable' in str(e):
+                    logger.error(f'Giving up on downloading comments because video is unavailable: {url=}', exc_info=e)
+                    break
+                logger.error(f'Got error when attempting to download video comments: {url=}', exc_info=e)
+                await asyncio.sleep(random.randint(1, 5))
+
+        if not info or not isinstance(info.get('comments'), list):
             logger.error(f'Never got comments for video: {url=}')
-            download_manager.add_to_skip_list(url)
             add_video_to_skip_list(url)
             continue
 
diff --git a/pi-gen/stage2/04-wrolpi/01-packages b/pi-gen/stage2/04-wrolpi/01-packages
index d35e5d4e..a2994f26 100644
--- a/pi-gen/stage2/04-wrolpi/01-packages
+++ b/pi-gen/stage2/04-wrolpi/01-packages
@@ -8,6 +8,7 @@ bzip2
 ca-certificates
 calibre
 cargo
+catdoc
 chromium-browser
 chromium-chromedriver
 cpufrequtils
diff --git a/requirements.txt b/requirements.txt
index e90d90a8..ef959a24 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,6 +16,7 @@ pypdf==3.17.0
 pytest-asyncio==0.21.0
 pytest==7.3.1
 python-dateutil==2.8.2
+python-docx==1.1.0
 python-dotenv==1.0.0
 python-magic~=0.4.27
 pytz==2023.3
diff --git a/test/example word.doc b/test/example word.doc
new file mode 100644
index 0000000000000000000000000000000000000000..ea44327898956b6a80aadf50f7bf98f17d45e3f3
GIT binary patch
literal 9216
zcmeI1TWnNC7{}*qw>PAvY_TGs9LfsFrSu}$3dpu>p%hCC#iHUR+wLN(?e5x^Sd0%O
z#>99@N;L7sL`^U;(HPW6O*Ankz8HCs7<?-6$;2oaHQ|BQ-~a46=~|$>T@3=++5C3y
z^UZwoojEgePJg#>;@yvzoHH}r7E@#<r^-!9M!H3Kffuhb#z$CblT%YuUNXUgyMZ)t
z$?dD=(9a|mgXX*x%Ag$P!7VT!DxeZn9usi<8)&$NuFlZFev{-kYHZVP68OXB7_*wY
ztHEM2o6MzV=6aD!`D|qh@~1VBO;^g#)W5fW&-MA&dqB_sLI^-LEP`9125y7Ja62r4
zrLYVn=T7|PV7u{J{FM-dyI>WphBa_Etc7*39yUN7Y=nAffJSJ7dtei6hI^qITHrp|
z0$X7l+z;C!1g+2pJ0J}0uoHGc2keGU*aHth7wm;@=z)E(A9~>cJP3Vo5FUbuL1R9I
z|EL?Do~{2sWYN1K7I^ifq+D-wW4?6$%}&i<SA1YNIg&g)Y9CAvM>m9%PahqMB}Sch
zLRYWS!pQ;05)`f@;^fygn?G7Uc_!V^Z2#C_TIg(rX2}a+b@SMT9Gc#lm_bbQ!jw-w
z@3OtDZ^LHDM4Y(nu_$OFLDLYl)&{NWlh`e^Vth@eZLR6<EHe9GN9STQ)KTsqfju2Q
z)6?lUi4K3+C_LU-Y$D`8(b3|crEjvN$>o;uS+b~7>?uYsq9)Et`y?q9^tt-vdE}3o
zlAy)&(%o5t@DhYCa|EUZEyJ?gs??4wyc|1=nK5V0ZewgwXT`2!4g%3dmPLyi(W{iS
z&BiLr3RpH_Kh_&JBh-?@7ZO+7AyZ4cwN{AKQl5c`Nl_wBs|ivnc(SYMoBB7O{skPR
zY%a6B@jq5(t#U@#h_AiFZt6vy-YHh?)JjVt+tdR3()PizH*GM&pgo5dE>Re#vT{Bx
zGIC1bfuneObF!;u9NWq64ltY{x-^11_ToQ_ua}+-LWA5yJHG7A@PHqqPCqqeR!ysN
zlirHy#a+{jBgxWwnZ;?M`swK~^}NZjk*}1vIZW+rW7=MVq%2aD9j48vkkV(4qLp4p
zBO|1@<XC?vJ&N(0uGy<GdQ&z_53V*w&53%Dskb(a#wvS_)#z&`>WDM2mutUT{R&7#
zdZaQU8+^`s;x{jhYgYf6>do^HKBuT-jT9=lJT(xg&$Hqz)x0@!i`cUi&CHr}PmX8S
z&ObcK$(NP+vIb<76ED8~`^D}fRUe!vGwW7=dRBdW2TDQtAy7^2Gjz_1g3ekg(0S{5
zm=7<3&RuVS%Kv=H8112|p?xeelp2iLym-g#1M$(pSUOMTv^NTY=R#fpU*Y2?o;%l<
z_|aMYkAL>tUkYaw$tn2DdzRH@m$R&&4}6iXk)8?`YOU>+W=AAB7)dl=VS>v2HD;Uh
zHb_^j%_4DXYJu}WtCI@rUEzgiR+Y|-dEXt6=g%ya8CRJ_nF(3KpHp75^Ns6!#s4J!
zEBssceV%RnRAfxQrr&iFw4+^l!QGr1ke$d5ueI};v)5W=L0&-v1r1zR8ffPv8sT7+
z;#d?jHuoMnZAS@(2@P<t(BVrrixtihW`I<k5p8NjI0eU?oy~PMq(W!rq5*fGH<v1f
zvNw$eEM3T~cWvNe=CCegMhf8@SOXka(>JE2L8~>WQ&&1l*!Z@Xt@S3<!zFr;&n!Q6
zdZT@6V%v&S34blD98dUz&`5s6c+}LVq=qJ9YZ^^uy&_jTKkESFJM-<Cx9XNwz5XWW
z=k*sq3X@hmBTaYex~FCn>`;ZTx;i5xhp*UaXXB|%Gbld<iYu(Mc55ot7aJalC%F^!
z>g$Y+X`6;kb@ezxH=p}2zyC>}Gru8RNn@ImkUkLMEN-238JpsZN<Wv8b9Dh`OY_AA
zMEXuS#|At}7IL+7$=A(8y1t!gO!9MSNB#9~8XCCFjWF_j`%vl1mu$IQ<g2gk?nQes
zyE5(3bRDj5zq-Lc?Cjx`;$1&qz120y*Cnf(%T@jzls)Y>>FpP1?iStbtlpA5#=nAA
ky?dwP%l4^r(?@e2@*hF|hgnCKdjV(S-M?IFWq$wt8>s2++W-In

literal 0
HcmV?d00001

diff --git a/test/example word.docx b/test/example word.docx
new file mode 100644
index 0000000000000000000000000000000000000000..8ab4f0660f4a1c16b7c2457345ed28d81303245b
GIT binary patch
literal 4322
zcmaJ^2UJsS5)IO;p*N)?Nbev`nh1mvA%IBlB@jv?9i&K?-cgW_)P&x97o><FRVf0Z
zL_k4MKsNaQyYjoc&O7Iw<eYbt_uZMfa|aB<#-Rcb5D)<DqipU1E(9^=v$=<@Gh9&M
z{9c)~q=BtS61H<prTnRN&H$L0jgBQ~r!2h;w5w@Vmbq!8TIssO5-pH&t9yilYt#QB
zqP6yBDQ()e)db4%9{7r8oxmf89J0fYjI#&jpAvN-WgI#Jw)$R5GeW6JF!I{WSv{E5
zDws=|N~b7I#EK1GnCQOH@S6LkT<%k!vi4~fzgDdK-k=l49VwDARP4qaoST+y0?rBw
zn^0M~pX9{W?_NdE!A?th|2VJVOGJJtR#19#!-Mq}<K<QOip87&`yCCCis6~Mn4y1f
zn`-sZ{Z*YT{fJuE_iwSlAbf(1%6_6H%$Zd#0RZ6t%L+N>2^&{yu!pM~T+rIp!&boC
z#W_~@j!vi$F!0bYp+nhHv%m(WvYBIICGu#X@gA%~12p9sdg=$FcNX;%+Ui|83^@H%
z^?I&tJ+%?P$cSH;3YYb>!bFQjzUHKH$No-Y2`M>lsT0<9=^)iF+jgq9N>E&6Sz0a$
zHrj|ikCI2PPF(dG9kc~k8(~4+ll1tWyOnd91|Q)mE9bI0-5X1?aH>AwBK>y{gv#xa
zkhnCD5*~Xnn1tF5sJg}-xvLIUa1S^Lu^vl#k{SB2{AKW!=}Nl-lw($IRQ9W{%q#KJ
zBlEEzL?`8_aZ_Hz#d1Kfgu&i3EL$U_qkboKL+RX)A10SF4+KiRC=jaV3OstUa`RB8
zSH4Prrn;nW0N0@`L(DTm&IfC@jV*t14u4$X$Q8;(p9f2&p0u+~@2~{RGF1#xJHL-)
zu0D&W;+#a!sf7lYS6Ga<bf({`dY|EG`f=&CBzP)b?3#qcC?r9y@!<$j+bu0)!5tQ^
zZS_shZ_2gi=lQZ1;Vf{+0szo|%op%tzAW9`&eN5hI0EYt0^T}2jUKWBUC}kW6?cdE
z15v4(R-Vi|Y2{U(YJNY3-9zz*#YV5oo8iduqwchl;<&iwHW6v|0x4E<2S=i!kw^aT
zki9peWQj~8=N*QI=w*#Q%QScngec?66XBpmby??ei_8or2$=DW@$Fqr*oD*#b!*@9
z5(=s^F-)jUab)UBvA(|)orex8<P?ZIGIy|ix9%U9{M2RHJx$pf8r(&H$_o$8AJfN{
zB}8kZZbPA*xZ$)bya(&cXz7r#Rd?B#scwy}UdO{5Pd^o}J)@ls<Mj_!k=Dl3!Plql
zO_`M6R(eyX^Wt{06yZmPBXo5=clQ@mIsfcTQ_mN|0+;R_-j#=az|Ae}0=c=`2BLB=
zUK_w-$tif8Ao&S*Hfq|%q$K3Me8?H58}JFdk#@F17`B=I$o3Ts+g`37HiCZ%2c}W(
zyV$}I=cNB7B8gf=80NzcbJ#}T6zc;Mi(x8myaxc`?1pX$!eMS-KkVpx+^aq7CjmRc
zg68JOai$aaF0ny;>cM$;8eSHYyKy&8aaP2rcMNLlnh;ic2KO^k;?$|Y`hCFsUqv`(
z@}iRGznkgu1@vY^xaL@%7Hx$b>)=sgUt7?>bW-&^C{;3|6XjwG4Pz`HlVe~=Tn>g^
z4OdI)P^8#>mOO+dFL3j?#NUN9_4Dk?LU;MWi@p=&O@;i7^-bEG?2<6`G5i%8uX8%S
z2?}-fV-8FCM>@{M;wK#!iRg!b;vWkEI}Qz5-cqu>FyO7T5p*nPYKcyP$T3=XRhrjm
zseS(_LndM4Rm6j4^6j1#{UV1qa5<i<JDFWP?YdFEe`el8jeMtT3lsq|{`VFO`lR`w
zqp5|Q(t0sWk~>V{`38lwkQN+ZAwML_mz@|S&4V%}P<>QW=1!cil9v@u&Ok%OqZuW=
ziAXHYb0`%{1wvEEXS-+Ww(i!Dnr!im$QOotY%3tq84cv<Bd#9>%fhZrVfVqsGpgiH
z{9;PGiG~)}xH4p0yHSIgJp25k_1(SH>u}ZuVJY@`!-V8VoI?alt+@cRoc68-ue($9
zl>v<N_65FICvlqlE4w!AkpY8Z{*%$}8)LL<wa@%dCG-XTzPfrFp6LW0dI8D_6%tlK
zys6#P5$cyXqZ-ozZ1Y_T`epLR`sPeBaHFrEMjb_{ZjXZY-x&|y@g24kkJ+5_-Vg1=
z&bYfapu86R(ucHbL*JQuabt~;nAHERqB}*zn0=G5Ygq*exVaXOnW6EOUhJOOhqDLQ
zzNT;+ahbgq-MW)i)E8zV1f1iLYc@q)ibDUORaveXGAtK2iB_KMK3+PNjv+L!-oYw~
z4ORFNh=!9uTgzP*WNybcZhX96RT8V`s|`)bn&Mf1LcvP+EXUWKu1Mn2GE`&smC{XZ
znF5pjNefsgA%0rJXpm_WpY@Z+%GC0P&MFLN?TC(P$<CiM=rj&yO@}Ed2E0Gbpv(U*
zD>%Z(*%p4@UHuSK9Y$fG{|-{>o1BS4WPT*}WNxGycO}vO(jI!<<U}7C4-5=&*(720
zN<nJ0acmm-$jOM#7bMGEuQ5gq)$<H}8cVRr%@An5NA>QzlZX@ty>yudPhChz1z>o1
zFTG%$U`YdD_oJg{vWBG|ZkMt-<di_=Rfn|>O>Ib$eoz<VU+UsmVHfm%0hG8y8iC(j
zXp;EK&Z~1u&BIRo!*jF{KipLRiK6p5a))d!ovJk02Fc1++U_{obtw93?*{E+B^!N`
zx<r;hN&`aRq%P<3jjy$7S=oxW7*~X@pf4{lE$E#*c32lOj^h>?BlhUtv<UF4tO23g
z*g7CjdHjoUsFtpK+)EHW(t031$Ei<8L_#c54cL1Ukq+zPF>fl$XYqp)>@WwNK78cP
zND1Mmz;kj7__n`t>?@Rm>o5iyuh|IQ;xidtp#Op%e)0AC3BONix-~Lv+UiB2@PO~Q
z4r9r<VKLr;!sr~{Jgw$&kKK4%fj>!hC)!|O!TMPdF-YsG+ThDNz9Bv-{(RSH4K>$#
zx59mUim8ON7*8PU(p6Z}3C>_9&HOcyS09hI6eFEFXA<GLmU#$1^`m>P(cnPC0MF}9
zaKg^dZ{H%CeP@Ok0ip{{2_hTc=$?CD@h1zssxV<q7y0se@WF!X4qCo3?O=A%;LY;q
zaIXltH&+AuLh$Dr<OnQi$^=^`qCY>&UnUBau+$k)YlyV}eh}mwd7`p<Dj)S$<`wSk
z*Yw;W0kyJPV*?#qwZ044hq_nvC|4&eGk4Qfb_l)aOK8oujo1TLcQd^s6gpYoEUA1x
z*r79TH-1w-1xa;#eaA8Ohh(h0mDl&4wa#-vbg@Dd1YjyxjPOsDOY!f@wR43bAeL6n
zKkZXx(l}-wlML+mMRR+pDb9=OQ;9jwJFW`NVrMMBREyH<4}R2W?x2{Zdil7Qf6xBG
zkGWyzy_E_n3yZbVq-1wi+@!C@%B>n5NORSAT{oQhkb{S*vTP&$C$O~V^LjRkBT*xe
z;cEj8km&oNWZv7xg%#8CQjy;nJbAvMTtZ(DJz2`upUU?JlaUP9Y8^$zBB2F)b06E4
zqubRLh=p~!VjYdf#u03(I0DMZSY;ZJ=IRTXTmeN%Af3()d{ipm=j8w(q7T(`ci6_^
zBW3)xspC>eHm{U*e0tVI=6xk-K1^T_8wZ;)hD-f`+fc2%V8L5P&Et!i&JEk9Y3A?^
zn~XDp&rT9_EfEyFB*mXHx&~|w1m)kp)gX1(scJr~9(!;w@c5XBq))l+)Z<L+TX}uX
zW!W>FbKb}JV#A~`bGZ}yPrUzU5W#H`2uGOxPtC9ex59dbNEVN&@ec242g_Tjj2R%d
zQueW_GUIU7@>wR`r;vhr?7I`O@0Rww7tx)eNP&!)f@hX`l3>faSFPSNyZ8Mc_=lFE
zqT2VU?(|X<Z?{G)Ez;P~7?j#5^Pwv&ZL}+O`{U>|ORrl}Ip<y>0X-gfDqbwnqydpT
z?tUnBG=m5UC~@&Vh*T19B&_WpVrU?Jw$L`%t(g<Qfw0i%X`(}})Xxc^GBf0>9Qz#Z
zS}nSdQ3~LISn18mdp%l!1%wn%nJvj*;*d{Yu@i{d9t!HZN$BMTLle0f@a*>lsTQ>)
z$0b4azGVk`OM>Tg83NKNcQHi>!%+7BO+F>&31ekUiDJBqImE}!7H)bjvh67&+P#1K
zM&pzwlmY?gR>&Yl3jx$-Tiq#=xn}9*54kn5GB9=4r7&Ar_)ccCcGCDvnU=wFn^+6W
z=3EL#ZZo;AB(6`{rzu_$UM`>d>SfC9=?<0{;I?>Cj-Z0@Q+i?s-g@_FTz>><{iDum
zy!ZVGU-jlmEQFf!K8tN2vAA&qYqRm%URsPt(xUS*TN%sfynzP|O4B-(?8BA1A3<(K
z{cjMCH;uxdR}Bgl9@v{xR=~ajbo*bdenYKJ_n-=Sb2rG>JarXB>;+_n!btaTz?5n!
zcSKbv!pF0#nUMX8o>E8HZO^p%8G?x^;@I;K4ugxUIPv%I@}$@RLupvzY&sdazPowR
z%>H0j^m;!&VUE7K<Yr6uX00TOahoSvT~0dU@?)Blt&-XAajQ@jgYBZzZemG^?lE6k
zYyvFt2Z>5Mi}6Ckj)$FmH-CCd;%V|h2qqzBxaUd1x<m!|wUJ)j%FZ{^f5wIV^mpLJ
z_29hxe+3QZul{R=`yGDK@1E}vzhVHRa{qz<y;1y*zGx56J@Ky)!}+sA{{5FOI)Z;5
zLm9)?f0@hQ;TL83+&27*Ed0OWzf8pM;EURFuH3%@Li9KIA3FX!{$jKJ^W8@=sr>ow
m7xn0O_{Fj~@5o<qO!7bb6AZ$|_$B~=5Oc|5vXnq}{_S6sHwN?o

literal 0
HcmV?d00001

diff --git a/wrolpi/common.py b/wrolpi/common.py
index 0734b750..54ec84bd 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -23,7 +23,7 @@ from itertools import islice, filterfalse, tee
 from multiprocessing import Lock, Manager
 from pathlib import Path
 from types import GeneratorType
-from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set
+from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set, Coroutine
 from urllib.parse import urlparse, urlunsplit
 
 import aiohttp
@@ -1183,13 +1183,24 @@ def chunks_by_stem(it: List[Union[pathlib.Path, str]], size: int) -> Generator[L
 
 
 @contextlib.contextmanager
-def timer(name):
+def timer(name, level: str = 'debug'):
     """Prints out the time elapsed during the call of some block."""
     before = datetime.now()
+    log_method = getattr(logger_, level)
     try:
         yield
     finally:
-        logger_.warning(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
+        log_method(f'{name} elapsed {(datetime.now() - before).total_seconds()} seconds')
+
+
+def async_timer(coro: Coroutine, name: str = 'async timer', level: str = 'debug') -> callable:
+    """Returns a new coroutine which prints out time elapsed when calling the coroutine."""
+
+    async def _():
+        with timer(name, level):
+            return await coro
+
+    return _()
 
 
 @contextlib.contextmanager
@@ -1270,7 +1281,7 @@ def limit_concurrent(limit: int, throw: bool = False):
     return wrapper
 
 
-def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> Union[List[str], str]:
+def truncate_object_bytes(obj: Union[List[str], str, None, Generator], maximum_bytes: int) -> Union[List[str], str]:
     """
     Shorten an object.  This is useful when inserting something into a tsvector.
 
diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py
index 99ababdf..750d96f3 100644
--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@@ -388,6 +388,20 @@ def example_mobi(test_directory) -> pathlib.Path:
     yield destination
 
 
+@pytest.fixture
+def example_doc(test_directory) -> pathlib.Path:
+    destination = test_directory / 'example word.doc'
+    shutil.copy(PROJECT_DIR / 'test/example word.doc', destination)
+    yield destination
+
+
+@pytest.fixture
+def example_docx(test_directory) -> pathlib.Path:
+    destination = test_directory / 'example word.docx'
+    shutil.copy(PROJECT_DIR / 'test/example word.docx', destination)
+    yield destination
+
+
 @pytest.fixture
 def make_files_structure(test_directory) -> Callable[[Union[List[Union[pathlib.Path, str]], dict]], List[pathlib.Path]]:
     """
diff --git a/wrolpi/dates.py b/wrolpi/dates.py
index e2296213..7457f8f2 100644
--- a/wrolpi/dates.py
+++ b/wrolpi/dates.py
@@ -1,7 +1,7 @@
 import logging
 from datetime import datetime, timezone, timedelta
 from enum import Enum
-from typing import Union
+from typing import Union, List
 
 import dateutil.parser
 import pytz
@@ -192,3 +192,44 @@ class TZDateTime(types.TypeDecorator):
             # Assume the DB timestamp is UTC.
             value = value.replace(tzinfo=pytz.utc) if not value.tzinfo else value
         return value
+
+
+def months_selector_to_where(months: List[int], wheres: list, params: dict):
+    """Filter FileGroup results by the month that the file was published.  Returns wheres/params unchanged if months
+    is empty.
+
+    @param months: List of integers representing the index of the month of the year (January is 1).
+    @param wheres: A list of strings which are SQL where filters.
+    @param params: Dict which contains the parameters passed to SQLAlchemy query.
+    """
+    if not months:
+        return wheres, params
+
+    if not set(months).issubset(set(range(1, 13))):
+        raise ValueError('Months must be between 1 and 12')
+
+    wheres.append('EXTRACT (MONTH FROM published_datetime) IN %(months_list)s')
+    params['months_list'] = tuple(months)
+    return wheres, params
+
+
+def date_range_to_where(from_year: int, to_year: int, wheres: list, params: dict):
+    """Filter FileGroup results by the year that the file was published.
+
+    @param from_year: The earliest year the file was published
+    @param to_year: The latest year the file was published
+    @param wheres: A list of strings which are SQL where filters.
+    @param params: Dict which contains the parameters passed to SQLAlchemy query.
+    """
+    if from_year and to_year:
+        wheres.append('EXTRACT (YEAR FROM published_datetime) BETWEEN %(from_year)s AND %(to_year)s')
+        params['from_year'] = from_year
+        params['to_year'] = to_year
+    elif from_year:
+        wheres.append('EXTRACT (YEAR FROM published_datetime) >= %(from_year)s')
+        params['from_year'] = from_year
+    elif to_year:
+        wheres.append('EXTRACT (YEAR FROM published_datetime) <= %(to_year)s')
+        params['to_year'] = to_year
+
+    return wheres, params
diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py
index a55585af..ae6e8dad 100644
--- a/wrolpi/files/api.py
+++ b/wrolpi/files/api.py
@@ -100,7 +100,7 @@ async def refresh_progress(request: Request):
 @validate(schema.FilesSearchRequest)
 async def post_search_files(_: Request, body: schema.FilesSearchRequest):
     file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
-                                          body.tag_names, body.headline)
+                                          body.tag_names, body.headline, body.months, body.from_year, body.to_year)
     return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))
 
 
diff --git a/wrolpi/files/downloader.py b/wrolpi/files/downloader.py
index 367fd04b..e02dc29f 100644
--- a/wrolpi/files/downloader.py
+++ b/wrolpi/files/downloader.py
@@ -55,11 +55,12 @@ class FileDownloader(Downloader, ABC):
             background_task(save_and_tag(output_path, download.settings.get('tag_names')))
 
             with get_db_session(commit=True) as session:
-                FileGroup.from_paths(session, output_path)
+                fg = FileGroup.from_paths(session, output_path)
+                location = fg.location
 
             return DownloadResult(
                 success=True,
-                location=f'/download/{output_path.relative_to(media_directory)}'
+                location=location,
             )
         except Exception as e:
             logger.error(f'Failed to download {repr(str(download.url))}', exc_info=e)
diff --git a/wrolpi/files/indexers.py b/wrolpi/files/indexers.py
index 782f12c1..0347861d 100644
--- a/wrolpi/files/indexers.py
+++ b/wrolpi/files/indexers.py
@@ -1,9 +1,12 @@
 import pathlib
+import subprocess
 from abc import ABC
 from collections import defaultdict
 from typing import List, Type, Tuple
 from zipfile import ZipFile
 
+import docx
+
 from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
 
 try:
@@ -135,3 +138,38 @@ class HTMLIndexer(Indexer, ABC):
         title = metadata.title or get_title_from_html(contents)
 
         return title, a, metadata.description, words
+
+
+@register_indexer('application/msword')
+class DocIndexer(Indexer, ABC):
+    """Extracts words from old Doc files."""
+
+    @classmethod
+    def create_index(cls, path: pathlib.Path):
+        a = cls.get_title(path)
+
+        cmd = ('catdoc', str(path.absolute()))
+        proc = subprocess.run(cmd, capture_output=True)
+        text = proc.stdout.decode()
+        text = truncate_object_bytes(text, FILE_MAX_TEXT_SIZE)
+        words = split_lines_by_length(text)
+
+        return a, None, None, words
+
+
+@register_indexer('application/vnd.openxmlformats-officedocument.wordprocessingml.document')
+class DocxIndexer(Indexer, ABC):
+    """Extracts words from Docx files."""
+
+    @classmethod
+    def create_index(cls, path: pathlib.Path):
+        a = cls.get_title(path)
+
+        doc = docx.Document(str(path))
+        text = ' '.join(truncate_object_bytes(
+            (i.text for i in doc.paragraphs),
+            FILE_MAX_TEXT_SIZE)
+        )
+        words = split_lines_by_length(text)
+
+        return a, None, None, words
diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py
index 06569fa0..89250a0f 100644
--- a/wrolpi/files/lib.py
+++ b/wrolpi/files/lib.py
@@ -23,8 +23,9 @@ from wrolpi import flags
 from wrolpi.cmd import which
 from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
     partition, cancelable_wrapper, \
-    get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, get_config
-from wrolpi.dates import now, from_timestamp
+    get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, get_config, \
+    timer
+from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
 from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
 from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
     NoPrimaryFile, InvalidDirectory
@@ -540,7 +541,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
 
     refreshing_all_files = False
 
-    with flags.refreshing:
+    with flags.refreshing, timer('refresh_files', 'info'):
         if not paths:
             refresh_logger.warning('Refreshing all files')
             refreshing_all_files = True
@@ -684,21 +685,22 @@ async def search_directories_by_name(name: str, excluded: List[str] = None, limi
 
 
 def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
-                 tag_names: List[str] = None, headline: bool = False) -> \
+                 tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
+                 from_year: int = None, to_year: int = None) -> \
         Tuple[List[dict], int]:
     """Search the FileGroup table.
 
     Order the returned Files by their rank if `search_str` is provided.  Return all files if
     `search_str` is empty.
 
-    Parameters:
-        search_str: Search the ts_vector of the file.  Returns all files if this is empty.
-        limit: Return only this many files.
-        offset: Offset the query.
-        mimetypes: Only return files that match these mimetypes.
-        model: Only return files that match this model.
-        tag_names: A list of tag names.
-        headline: Includes Postgresql headline if True.
+    @param search_str: Search the ts_vector of the file.  Returns all files if this is empty.
+    @param limit: Return only this many files.
+    @param offset: Offset the query.
+    @param mimetypes: Only return files that match these mimetypes.
+    @param model: Only return files that match this model.
+    @param tag_names: A list of tag names.
+    @param headline: Includes Postgresql headline if True.
+    @param months: A list of integers representing the index of the month of the year, starting at 1.
     """
     params = dict(offset=offset, limit=limit, search_str=search_str, url_search_str=f'%{search_str}%')
     wheres = []
@@ -721,6 +723,8 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
         wheres.append('model = %(model)s')
 
     wheres, params = tag_append_sub_select_where(tag_names, wheres, params)
+    wheres, params = months_selector_to_where(months, wheres, params)
+    wheres, params = date_range_to_where(from_year, to_year, wheres, params)
 
     if search_str and headline:
         headline = ''',
@@ -1077,25 +1081,28 @@ def mimetypes_to_sql_wheres(mimetypes: List[str], params: dict):
     return '', params
 
 
-async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str]):
+async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str],
+                                       months: List[int] = None, from_year: int = None, to_year: int = None):
     """
     Return FileGroup count of what will be returned if the search is actually performed.
     """
     wheres = []
     joins = list()
-    params = dict(search_str=search_str, tag_names=tag_names, mimetypes=mimetypes, url_search_str=f'%{search_str}%')
+    params = dict(search_str=search_str, tag_names=tag_names, mimetypes=mimetypes)
     group_by = ''
     having = ''
 
     if search_str:
         # Search by textsearch, and by matching url.
-        wheres.append('(fg.textsearch @@ websearch_to_tsquery(%(search_str)s) OR fg.url ILIKE %(url_search_str)s)')
+        wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
 
     wheres, params = tag_append_sub_select_where(tag_names, wheres, params)
 
     mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params)
     if mimetype_wheres:
         wheres.append(mimetype_wheres)
+    wheres, params = months_selector_to_where(months, wheres, params)
+    wheres, params = date_range_to_where(from_year, to_year, wheres, params)
 
     joins = "\n".join(joins)
     wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py
index 95060118..f2b4547a 100644
--- a/wrolpi/files/models.py
+++ b/wrolpi/files/models.py
@@ -1,6 +1,7 @@
 import copy
 import pathlib
 import shutil
+import urllib.parse
 from datetime import datetime
 from typing import List, Type, Optional
 
@@ -9,10 +10,10 @@ from sqlalchemy import types
 from sqlalchemy.orm import deferred, relationship, Session
 from sqlalchemy.orm.collections import InstrumentedList
 
-from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory
+from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
+    get_relative_to_media_directory
 from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
 from wrolpi.db import optional_session
-from wrolpi.downloader import download_manager
 from wrolpi.errors import FileGroupIsTagged
 from wrolpi.files import indexers
 from wrolpi.media_path import MediaPathType
@@ -336,6 +337,18 @@ class FileGroup(ModelHelper, Base):
         # Flush the changes to the FileGroup.
         Session.object_session(self).flush([self, ])
 
+    @property
+    def location(self):
+        """Returns the URL that the FileGroup can be previewed."""
+        parent = str(get_relative_to_media_directory(self.primary_path.parent))
+        preview = str(get_relative_to_media_directory(self.primary_path))
+        if parent == '.':
+            # File is in the top of the media directory, App already shows top directory open.
+            query = urllib.parse.urlencode(dict(preview=str(preview)))
+        else:
+            query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
+        return f'/files?{query}'
+
 
 class Directory(ModelHelper, Base):
     """A representation of a file directory in the media directory."""
diff --git a/wrolpi/files/schema.py b/wrolpi/files/schema.py
index 8891e531..41ce6773 100644
--- a/wrolpi/files/schema.py
+++ b/wrolpi/files/schema.py
@@ -26,6 +26,9 @@ class FilesSearchRequest:
     model: Optional[str] = None
     tag_names: List[str] = field(default_factory=list)
     headline: bool = False
+    months: Optional[List[int]] = None
+    from_year: Optional[int] = None
+    to_year: Optional[int] = None
 
 
 @dataclass
diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py
index 94f381e9..6d6ad79d 100644
--- a/wrolpi/files/test/test_lib.py
+++ b/wrolpi/files/test/test_lib.py
@@ -176,6 +176,20 @@ async def test_refresh_files(test_session, make_files_structure, assert_file_gro
     assert_file_groups([{'primary_path': 'bar.txt', 'indexed': True}])
 
 
+@pytest.mark.asyncio
+async def test_file_group_location(test_session, make_files_structure):
+    """FileGroup can return the URL necessary to preview the FileGroup."""
+    make_files_structure({
+        'foo/one.txt': 'one',
+        'two.txt': 'two',
+    })
+    await lib.refresh_files()
+
+    one, two = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
+    assert one.location == '/files?folders=foo&preview=foo%2Fone.txt'
+    assert two.location == '/files?preview=two.txt'
+
+
 @pytest.mark.asyncio
 async def test_refresh_bogus_files(test_session, make_files_structure, test_directory, assert_file_groups,
                                    insert_file_group):
@@ -945,3 +959,53 @@ async def test_html_index(test_session, test_directory, make_files_structure):
 Some Text
 Span Text
 Outside element.'''
+
+
+@pytest.mark.asyncio
+async def test_doc_indexer(test_session, example_doc):
+    """The contents of a Microsoft doc files can be indexed."""
+    await lib.refresh_files()
+    assert test_session.query(FileGroup).count() == 1
+    doc = test_session.query(FileGroup).one()
+    assert doc.title == 'example word.doc'
+    assert doc.a_text == 'example word doc'
+    assert doc.d_text == 'Example Word Document\n\nSecond page\n'
+
+
+@pytest.mark.asyncio
+async def test_docx_indexer(test_session, example_docx):
+    """The contents of a Microsoft docx files can be indexed."""
+    await lib.refresh_files()
+    assert test_session.query(FileGroup).count() == 1
+    doc = test_session.query(FileGroup).one()
+    assert doc.title == 'example word.docx'
+    assert doc.a_text == 'example word docx'
+    assert doc.d_text == 'Example Word Document Second page'
+
+
+@pytest.mark.asyncio
+async def test_file_search_date_range(test_session, example_pdf, example_doc):
+    """Test searching FileGroups by their published datetime."""
+    await lib.refresh_files()
+    doc, pdf = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
+
+    files, total = lib.search_files('', 10, 0)
+    assert total == 2, 'Should only be 2 files.'
+
+    # PDF was published in December.
+    files, total = lib.search_files('', 10, 0, months=[12, ])
+    assert total == 1, 'Only PDF should be from December 2022'
+    assert files[0]['id'] == pdf.id
+
+    # PDF was published in 2022.
+    files, total = lib.search_files('', 10, 0, from_year=2022, to_year=2022)
+    assert total == 1, 'Only PDF should be from December 2022'
+    assert files[0]['id'] == pdf.id
+
+    # PDF was NOT published in 2023.
+    files, total = lib.search_files('', 10, 0, from_year=2023)
+    assert total == 0, 'No example files are published in 2023'
+
+    files, total = lib.search_files('', 10, 0, to_year=2022)
+    assert total == 1
+    assert files[0]['id'] == pdf.id
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 94b4daee..9111fa21 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -31,7 +31,7 @@ from wrolpi.errors import WROLModeEnabled, APIError, HotspotError, InvalidDownlo
 from wrolpi.events import get_events, Events
 from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
 from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
-    truthy_arg
+    truthy_arg, IS_RPI, IS_RPI4, IS_RPI5
 from wrolpi.version import __version__
 
 logger = logger.getChild(__name__)
@@ -384,6 +384,9 @@ async def get_status(_: Request):
         cpu_info=s.cpu_info,
         disk_bandwidth=s.disk_bandwidth,
         dockerized=DOCKERIZED,
+        is_rpi=IS_RPI,
+        is_rpi4=IS_RPI4,
+        is_rpi5=IS_RPI5,
         downloads=downloads,
         drives=s.drives,
         flags=flags.get_flags(),
@@ -515,11 +518,18 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ
     from modules.videos.channel.lib import search_channels_by_name
     from modules.archive.lib import search_domains_by_name
 
-    file_groups, channels, domains = await asyncio.gather(
-        search_file_suggestion_count(body.search_str, body.tag_names, body.mimetypes),
-        search_channels_by_name(body.search_str),
-        search_domains_by_name(body.search_str),
+    file_coro = search_file_suggestion_count(
+        body.search_str,
+        body.tag_names,
+        body.mimetypes,
+        body.months,
+        body.from_year,
+        body.to_year,
     )
+    channels_coro = search_channels_by_name(body.search_str)
+    domains_coro = search_domains_by_name(body.search_str)
+
+    file_groups, channels, domains = await asyncio.gather(file_coro, channels_coro, domains_coro)
 
     ret = dict(
         file_groups=file_groups,
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index 1a6ff6f8..8d1b93bd 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -143,3 +143,6 @@ class SearchSuggestionsRequest:
     search_str: Optional[str] = None
     mimetypes: List[str] = field(default_factory=lambda: list())
     tag_names: List[str] = field(default_factory=lambda: list())
+    months: Optional[List[int]] = None
+    from_year: Optional[int] = None
+    to_year: Optional[int] = None
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
index df0626c7..2c139f86 100644
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@@ -507,10 +507,10 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
     await assert_results(
         dict(search_str='foo l'),
         [
-         {'directory': 'Fool',
-          'id': 2,
-          'name': 'Fool',
-          'url': 'https://example.com/Fool'}],
+            {'directory': 'Fool',
+             'id': 2,
+             'name': 'Fool',
+             'url': 'https://example.com/Fool'}],
         [],
         0,
     )
@@ -555,6 +555,11 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
     # No PDFs.
     await assert_results(dict(mimetypes=['application/pdf']))
 
+    # Can filter by published datetime.
+    await assert_results(dict(months=[1, ]), expected_file_groups=2)
+    await assert_results(dict(months=[2, ]), expected_file_groups=0)
+    await assert_results(dict(from_year=2000, to_year=2000), expected_file_groups=2)
+
 
 def test_recursive_errors():
     """Errors are reported recursively."""
diff --git a/wrolpi/vars.py b/wrolpi/vars.py
index c5789d3e..44c6dad0 100644
--- a/wrolpi/vars.py
+++ b/wrolpi/vars.py
@@ -1,5 +1,6 @@
 import multiprocessing
 import os
+import pathlib
 import sys
 from pathlib import Path
 
@@ -80,3 +81,9 @@ VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 50
 DEFAULT_HTTP_HEADERS = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116 Safari/537.36',
 }
+
+RPI_DEVICE_PATH = pathlib.Path('/proc/device-tree/model')
+RPI_DEVICE_MODEL_CONTENTS = RPI_DEVICE_PATH.read_text() if RPI_DEVICE_PATH.is_file() else None
+IS_RPI4 = 'Raspberry Pi 4' in RPI_DEVICE_MODEL_CONTENTS if RPI_DEVICE_MODEL_CONTENTS else False
+IS_RPI5 = 'Raspberry Pi 5' in RPI_DEVICE_MODEL_CONTENTS if RPI_DEVICE_MODEL_CONTENTS else False
+IS_RPI = IS_RPI4 or IS_RPI5