From c23b014add9e597a26ac414dc0110c676a17f5cd Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Mon, 3 Jan 2022 15:13:02 -0700 Subject: [PATCH] Feature/archive search (#70) * Working through archive search. * Using custom hook. * Setting archive.contents duration creation. * Archive search and domain list use custom hooks. * Watching search param using value, not history. * Search params are preserved on load. Search and domain can be cleared. * Fixing archive external link. * Archive search form is new generic SearchInput. * Reworking Archives empty states. * Simplifying Archives logic. * Fixing bug which displayed the wrong empty state during loading. * Archives can be synced and deleted. * Changing wording of archive delete. * Removing URL table. * Deleting domain if it has no archives. * Fixing FE use of archive.url. Fixing bug with get_domains. * Universal results are displayed on Welcome page. * Creating ExternalLink to open a link in a new tab.g * Increasing size of search headers. * Adding divider to ease reading. * Adding More button below search results. * Disabling More buttons when there are no results. * Ordering videos by rank so linked results are at the top. Signed-off-by: Roland --- alembic/versions/53f6780aa599_drop_urls.py | 38 ++ app/src/App.css | 2 +- app/src/App.js | 77 ++- app/src/api.js | 10 +- app/src/components/Archive.js | 597 ++++++++------------- app/src/components/Common.js | 116 +++- app/src/components/Videos.js | 64 +-- app/src/hooks/useArchives.js | 52 ++ app/src/hooks/useDomains.js | 21 + app/src/hooks/useSearchParam.js | 21 + modules/archive/__init__.py | 2 +- modules/archive/api.py | 39 +- modules/archive/conftest.py | 16 +- modules/archive/lib.py | 245 +++------ modules/archive/models.py | 78 +-- modules/archive/schema.py | 25 - modules/archive/test/test_lib.py | 167 ++---- 17 files changed, 666 insertions(+), 904 deletions(-) create mode 100644 alembic/versions/53f6780aa599_drop_urls.py create mode 100644 app/src/hooks/useArchives.js create mode 100644 app/src/hooks/useDomains.js create mode 100644 app/src/hooks/useSearchParam.js diff --git a/alembic/versions/53f6780aa599_drop_urls.py b/alembic/versions/53f6780aa599_drop_urls.py new file mode 100644 index 00000000..9260add2 --- /dev/null +++ b/alembic/versions/53f6780aa599_drop_urls.py @@ -0,0 +1,38 @@ +"""Drop URLs. + +Revision ID: 53f6780aa599 +Revises: 31896cc29b71 +Create Date: 2022-01-03 12:25:51.575505 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +from modules.archive.models import Archive + +revision = '53f6780aa599' +down_revision = '31896cc29b71' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + urls = {i['id']: i['url'] for i in session.execute('SELECT id, url FROM url')} + archives = {i['id']: i['url_id'] for i in session.execute('SELECT id, url_id FROM archive')} + + session.execute('ALTER TABLE archive ADD COLUMN url TEXT') + + for archive_id, url_id in archives.items(): + archive = session.query(Archive).filter_by(id=archive_id).one() + archive.url = urls[url_id] + + session.execute('ALTER TABLE archive DROP COLUMN url_id') + session.execute('DROP TABLE url') diff --git a/app/src/App.css b/app/src/App.css index 8e71b651..17abf790 100644 --- a/app/src/App.css +++ b/app/src/App.css @@ -39,7 +39,7 @@ pre.wrap-text { text-decoration: none !important; } -.video-card-link { +.card-link { color: black; } diff --git a/app/src/App.js b/app/src/App.js index 75c6ce47..d8d8be49 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -1,32 +1,83 @@ -import React from 'react'; +import React, {useEffect, useState} from 'react'; import './App.css'; import {NavBar} from "./components/Nav"; -import {Route, Switch} from "react-router-dom"; -import {FavoriteVideosPreview, VideosRoute, VideoWrapper, ViewedVideosPreview} from "./components/Videos"; +import {Route, Switch, useHistory} from "react-router-dom"; +import {VideosPreview, VideosRoute, VideoWrapper} from "./components/Videos"; import Admin from "./components/Admin"; -import {Container, Divider, Header, Segment} from "semantic-ui-react"; +import {Container, Divider, Header} from "semantic-ui-react"; import 'semantic-ui-offline/semantic.min.css'; import {SemanticToastContainer} from 'react-semantic-toasts'; import 'react-semantic-toasts/styles/react-semantic-alert.css'; import {AppsRoute} from "./components/Apps"; import {InventoryRoute} from "./components/Inventory"; -import {ArchiveRoute} from "./components/Archive"; +import {ArchiveRoute, ArchivesList} from "./components/Archive"; import {Saver} from "./components/Upload"; +import {useSearchParam} from "./hooks/useSearchParam"; +import {searchArchives, searchVideos} from "./api"; +import {MoreButton, SearchInput} from "./components/Common"; + +const useSearch = () => { + let [searchStr, setSearchStr] = useSearchParam('q'); + + const [archives, setArchives] = useState(); + const [videos, setVideos] = useState(); + + const localSearchArchives = async (term) => { + setArchives(null); + const [archives, total] = await searchArchives(0, 6, null, term); + setArchives(archives); + } + + const localSearchVideos = async (term) => { + setVideos(null); + const [videos, total] = await searchVideos(0, 6, null, term); + setVideos(videos); + } + + useEffect(() => { + localSearchArchives(searchStr); + localSearchVideos(searchStr) + }, [searchStr]); + + return {searchStr, setSearchStr, archives, videos} +} function Welcome() { - return ( - + const {searchStr, setSearchStr, archives, videos} = useSearch(); + const history = useHistory(); + + let body = ( + <>
Save your media
+ + ); + + if (searchStr) { + let archiveMore = () => { + history.push(`/archive?q=${searchStr}`); + }; + let videosMore = () => { + history.push(`/videos?q=${searchStr}&o=rank`); + }; + + body = (<> +
Archives
+ + - - - - - - +
Videos
+ + + ); + } + + return ( + + + {body} ) } diff --git a/app/src/api.js b/app/src/api.js index d2517e08..b3c2665c 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -307,16 +307,11 @@ export async function deleteItems(itemIds) { await apiDelete(`${API_URI}/inventory/item/${i}`); } -export async function postArchive(url) { - let i = {url: url}; - return await apiPost(`${API_URI}/archive`, i); -} - export async function deleteArchive(archive_id) { return await apiDelete(`${API_URI}/archive/${archive_id}`); } -export async function searchArchives(offset, limit, domain = null) { +export async function searchArchives(offset, limit, domain = null, searchStr = null) { // Build a search query to retrieve a list of videos from the API offset = offset || 0; limit = limit || DEFAULT_LIMIT; @@ -324,6 +319,9 @@ export async function searchArchives(offset, limit, domain = null) { if (domain) { body['domain'] = domain; } + if (searchStr) { + body['search_str'] = searchStr; + } let response = await apiPost(`${ARCHIVES_API}/search`, body); if (response.status === 200) { diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 88dd52b6..5f174105 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -1,29 +1,31 @@ -import React from "react"; -import {Card, Confirm, Container, Form, Header, Icon, Image, Placeholder, Tab, Table} from "semantic-ui-react"; -import Paginator, {APIForm, objectToQuery, scrollToTop, uploadDate} from "./Common"; -import {deleteArchive, fetchDomains, postArchive, refreshArchives, searchArchives} from "../api"; +import React, {useState} from "react"; +import {Card, Confirm, Container, Icon, Image, Placeholder, Tab} from "semantic-ui-react"; +import Paginator, {ClearButton, ExternalLink, SearchInput, uploadDate} from "./Common"; +import {deleteArchive, postDownload, refreshArchives} from "../api"; import Button from "semantic-ui-react/dist/commonjs/elements/Button"; import {Link, NavLink} from "react-router-dom"; -import * as QueryString from "query-string"; import {ArchivePlaceholder} from "./Placeholder"; +import Table from "semantic-ui-react/dist/commonjs/collections/Table"; +import {useArchives} from "../hooks/useArchives"; +import {useDomains} from "../hooks/useDomains"; -function FailedArchiveCard({url, syncURL, deleteURL}) { +function FailedArchiveCard({archive, syncArchive, deleteArchive}) { let syncIcon = ( - ); let trashIcon = ( - ); let externalIcon = ( - + ); @@ -32,7 +34,7 @@ function FailedArchiveCard({url, syncURL, deleteURL}) { - {url.url} + {archive.url}

Failed!

@@ -45,345 +47,172 @@ function FailedArchiveCard({url, syncURL, deleteURL}) { ); } -class ArchiveCard extends React.Component { - constructor(props) { - super(props); - this.state = { - deleteOpen: false, - syncOpen: false, - } +function ArchiveCard({archive, syncArchive, deleteArchive}) { + const [syncOpen, setSyncOpen] = useState(false); + const [deleteOpen, setDeleteOpen] = useState(false); + + const localSyncArchive = async () => { + setSyncOpen(false); + await syncArchive(archive.url); + } + const localDeleteArchive = async () => { + setDeleteOpen(false); + await deleteArchive(archive.id); } - syncURL = async () => { - await postArchive(this.props.archive.url.url); - await this.props.fetchURLs(); + let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null; + let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null; + + if (archive.status === 'failed') { + return ; } - deleteURL = async () => { - await deleteArchive(this.props.archive.id); - await this.props.fetchURLs(); + let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null; + let readabilityIcon = ; + if (readabilityUrl) { + readabilityIcon = ( + ); } - render() { - let archive = this.props.archive; + let syncIcon = null; + if (syncArchive) { + syncIcon = (<> + + setSyncOpen(false)} + onConfirm={localSyncArchive} + /> + ); + } - let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null; - let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null; - - if (archive.status === 'failed') { - return ; - } - - let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null; - let readabilityIcon = ; - if (readabilityUrl) { - readabilityIcon = ( - ); - } - - let syncIcon = ( - <> - this.setState({syncOpen: false})} - onConfirm={this.syncURL} + open={deleteOpen} + content='Are you sure you want to delete this Archive? All files will be deleted.' + confirmButton='Delete' + onCancel={() => setDeleteOpen(false)} + onConfirm={localDeleteArchive} /> ); - - let deleteIcon = ( - <> - - this.setState({deleteOpen: false})} - onConfirm={this.deleteURL} - /> - - ) - ; - - let externalIcon = ( - - ); - - return ( - - - - - - -

{archive.title || archive.url}

-
-
-
- - {uploadDate(archive.archive_datetime)} - - - - {readabilityIcon} - {syncIcon} - {deleteIcon} - {externalIcon} - - -
-
- ) } + let externalIcon = ( + + ); + + return ( + + + + + + + + +

{archive.title || archive.url}

+
+
+
+ + {uploadDate(archive.archive_datetime)} + + + + {readabilityIcon} + {syncIcon} + {deleteIcon} + {externalIcon} + + +
+
+ ) } -class ArchiveCards extends React.Component { - render() { - return ( - - {this.props.archives.map((i) => { - return - })} - - ) +function ArchiveCards({archives, syncArchive, deleteArchive}) { + return ( + + {archives.map((i) => { + return + })} + + ) +} + +export function ArchivesList({archives, searchStr, syncArchive, localDeleteArchive}) { + if (archives === null || archives === undefined) { + // Fetching archives. + return ; + } else if (archives.length === 0 && searchStr) { + // Search with no results. + return

No archives found! Is your search too restrictive?

; + } else if (archives.length === 0) { + // No archives fetched. + return

No archives found! Have you archived any webpages?

; + } else { + // Archives fetched successfully! + return ; } } -class Archives extends React.Component { +export function Archives() { + const {archivesData, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} = + useArchives(); + const {archives} = archivesData; - constructor(props) { - super(props); - const query = QueryString.parse(this.props.location.search); - let activePage = query.page ? parseInt(query.page) : 1; - let domain = query.domain || null; - - this.state = { - activePage: activePage, - limit: 20, - archives: null, - totalPages: null, - domain: domain, - }; - this.fetchURLs = this.fetchURLs.bind(this); - this.clearSearch = this.clearSearch.bind(this); - this.changePage = this.changePage.bind(this); + const syncArchive = async (url) => { + await postDownload(url, 'archive'); + await search(); } - async componentDidMount() { - await this.fetchURLs(); + const localDeleteArchive = async (archive_id) => { + await deleteArchive(archive_id); + await search(); } - async componentDidUpdate(prevProps, prevState, snapshot) { - if (prevProps.location.search !== this.props.location.search) { - await this.fetchURLs(); - } + let domainClearButton = null; + if (domain) { + domainClearButton = setDomain(null)} + style={{marginBottom: '1em'}} + label={`Domain: ${domain}`} + />; } - async fetchURLs() { - this.setState({archives: null}); - let offset = this.state.limit * this.state.activePage - this.state.limit; - let [archives, total] = await searchArchives(offset, this.state.limit, this.state.domain); - let totalPages = Math.round(total / this.state.limit) || 1; - this.setState({archives: archives, totalPages: totalPages}); - } - - clearSearch() { - this.setState({activePage: 1, domain: null}, this.changePage); - } - - changePage(activePage) { - if (activePage) { - this.setState({activePage}); - } - let {history, location} = this.props; - - let search = { - page: activePage > 1 ? activePage : null, - domain: this.state.domain, - }; - - history.push({ - pathname: location.pathname, - search: objectToQuery(search), - }); - scrollToTop(); - } - - render() { - let {archives, activePage, totalPages} = this.state; - - if (archives === null) { - return (<> -
Latest Archives
- - ) - } - - let pagination = null; - if (totalPages) { - pagination = ( -
- -
- ) - } - - let domainButton = null; - if (this.state.domain) { - domainButton = ( - - ) - } - - return ( - <> -
Latest Archives
- {domainButton} - - {pagination} - - ) - } -} - -class Domains extends React.Component { - constructor(props) { - super(props); - this.state = { - domains: null, - total: null, - }; - this.fetchDomains = this.fetchDomains.bind(this); - } - - async componentDidMount() { - await this.fetchDomains(); - } - - async fetchDomains() { - this.setState({domains: null}); - let [domains, total] = await fetchDomains(); - this.setState({domains, total}); - } - - row(domain) { - return - - - {domain['domain']} - - - {domain['url_count']} - - } - - render() { - if (this.state.domains) { - return ( - <> -
Domains
- - - - Domain - URLs - - - - - {this.state.domains.map(this.row)} - -
- - ) - } - - return (<> -
Domains
- - - - - - - ) - } -} - -class ArchiveAddForm extends APIForm { - constructor(props) { - super(props); - this.archivesRef = props.archivesRef; - - this.state = { - ...this.state, - inputs: { - url: '', - }, - errors: {}, - }; - this.fetchURLs = this.fetchURLs.bind(this); - } - - handleSubmit = async (e) => { - e.preventDefault(); - this.setLoading(); - try { - let response = await postArchive(this.state.inputs.url); - if (response.status === 204) { - this.setState({inputs: {url: ''}, success: true}, this.fetchURLs); - } else { - this.setState({loading: false, success: undefined, error: true}); - } - } finally { - this.clearLoading(); - } - } - - fetchURLs = async (e) => { - await this.archivesRef.current.fetchURLs(); - } - - render() { - return ( - <> -
- - - - Archive - -
- - ) - } + return ( + <> + + {domainClearButton} + +
+ +
+ + ) } class ManageArchives extends React.Component { @@ -401,6 +230,51 @@ class ManageArchives extends React.Component { } } +function Domains() { + + const [domains] = useDomains(); + + const row = (domain) => { + return + + + {domain['domain']} + + + {domain['url_count']} + + } + + if (domains) { + return ( + <> + + + + Domain + URLs + + + + + {domains.map(row)} + +
+ + ) + } + + return (<> + + + + + + + ) +} + + export class ArchiveRoute extends React.Component { constructor(props) { @@ -410,7 +284,43 @@ export class ArchiveRoute extends React.Component { this.state = { activeIndex: this.matchPaneTo(), } - this.archivesRef = React.createRef(); + + this.archivePanes = [ + { + menuItem: { + as: NavLink, + content: 'Archives', + id: 'archive', + to: '/archive', + exact: true, + key: 'home', + }, + render: () => + }, + { + menuItem: { + as: NavLink, + content: 'Domains', + id: 'domains', + to: '/archive/domains', + exact: true, + key: 'domains', + }, + render: () => + }, + { + menuItem: { + as: NavLink, + content: 'Manage', + id: 'manage', + to: '/archive/manage', + exact: true, + key: 'manage', + }, + render: () => + }, + ]; + } componentDidUpdate(prevProps, prevState, snapshot) { @@ -424,46 +334,9 @@ export class ArchiveRoute extends React.Component { } render() { - const panes = [ - { - menuItem: { - as: NavLink, - content: 'Archives', - id: 'archive', - to: '/archive', - exact: true, - key: 'home', - }, - render: () => - }, - { - menuItem: { - as: NavLink, - content: 'Domains', - id: 'domains', - to: '/archive/domains', - exact: true, - key: 'domains', - }, - render: () => - }, - { - menuItem: { - as: NavLink, - content: 'Manage', - id: 'manage', - to: '/archive/manage', - exact: true, - key: 'manage', - }, - render: () => - }, - ]; - return ( - - + ) } diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 91f5c86f..02e0da02 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -1,6 +1,8 @@ -import React from "react"; -import {Card, Container, Image, Pagination, Progress} from 'semantic-ui-react'; +import React, {useState} from "react"; +import {Card, Container, Form, Icon, Image, Input, Pagination, Progress} from 'semantic-ui-react'; import {Link} from "react-router-dom"; +import Button from "semantic-ui-react/dist/commonjs/elements/Button"; +import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; export const API_URI = `http://${window.location.host}/api`; export const VIDEOS_API = `${API_URI}/videos`; @@ -99,7 +101,7 @@ export function VideoCard({video}) { let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null; return ( - + @@ -107,7 +109,7 @@ export function VideoCard({video}) { - +

{video.title || video.stem || video.video_path}

@@ -115,7 +117,7 @@ export function VideoCard({video}) { {channel && - + {channel.name} }

{upload_date}

@@ -126,17 +128,14 @@ export function VideoCard({video}) { ) } -export class VideoCards extends React.Component { - - render() { - return ( - - {this.props.videos.map((v) => { - return - })} - - ) - } +export function VideoCards({videos}) { + return ( + + {videos.map((v) => { + return + })} + + ) } export function RequiredAsterisk() { @@ -490,3 +489,88 @@ export function objectToQuery(d) { arr = arr.filter((i) => i !== '') return arr.join('&'); } + +export function ClearButton({onClick, style, label, icon = 'close'}) { + return +} + +export function SearchInput({initValue, onSubmit, size}) { + let [value, setValue] = useState(initValue || ''); + size = size || 'small'; + + const buttonClick = () => { + // Clear the input when the "clear" button is clicked, search again. + setValue(''); + onSubmit(value); + } + + // Button is "search" when input is dirty, otherwise it is "clear". + let button = ( + + ); + if (!initValue || initValue !== value) { + button = ( + + ); + } + + const localOnSubmit = (e) => { + // Send the value up when submitting. + e.preventDefault(); + onSubmit(value); + } + + return ( +
+ + setValue(e.target.value)} + value={value} + size={size} + /> + {button} + +
+ ) +} + +export function ExternalLink(props) { + return ( + + {props.children} + + ) +} + +export function MoreButton(props) { + return ( + + + + + + + + ) +} diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 060d9c97..83f091eb 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -137,62 +137,16 @@ export class VideoWrapper extends React.Component { } } -class VideosPreview extends React.Component { - constructor(props) { - super(props); - this.state = { - videos: null, - }; +export function VideosPreview({videos}) { + let body = + if (videos && videos.length === 0) { + body =

No videos available.

+ } else if (videos && videos.length > 0) { + body = } - - async componentDidMount() { - await this.fetchVideos(); - } - - render() { - let header =

{this.title}

; - let body = - if (this.state.videos && this.state.videos.length === 0) { - body =

No videos available.

- } else if (this.state.videos && this.state.videos.length > 0) { - body = - } - return <> - {header} - {body} - - } - -} - -export class FavoriteVideosPreview extends VideosPreview { - - constructor(props) { - super(props); - this.title = 'Favorite Videos' - } - - async fetchVideos() { - let [videos] = await searchVideos( - 0, 3, null, null, '-favorite', ['favorite']); - this.setState({videos}); - } - -} - -export class ViewedVideosPreview extends VideosPreview { - - constructor(props) { - super(props); - this.title = 'Recently Viewed Videos' - } - - async fetchVideos() { - let [videos] = await searchVideos( - 0, 3, null, null, '-viewed'); - this.setState({videos}); - } - + return <> + {body} + } function FilterModal(props) { diff --git a/app/src/hooks/useArchives.js b/app/src/hooks/useArchives.js new file mode 100644 index 00000000..a1b00ad5 --- /dev/null +++ b/app/src/hooks/useArchives.js @@ -0,0 +1,52 @@ +import {useEffect, useState} from "react"; +import {searchArchives} from "../api"; +import {useSearchParam} from "./useSearchParam"; + + +export const useArchives = (defaultLimit = 20) => { + const [archivesData, setArchives] = useState({archives: null, total: 0}); + const [totalPages, setTotalPages] = useState(0); + const [activePage, setActivePage] = useState(1); + + let [offset, setOffset] = useSearchParam('o'); + let [limit, setLimit] = useSearchParam('l', defaultLimit); + let [domain, setDomain] = useSearchParam('domain'); + let [searchStr, setSearchStr] = useSearchParam('q'); + + const setPage = (i) => { + i = parseInt(i); + let l = parseInt(limit); + setOffset((l * i) - l); + setActivePage(i); + } + + const search = async (term) => { + setArchives({archives: null, total: 0}); + setTotalPages(0); + const [archives, total] = await searchArchives(offset, limit, domain, term); + setTotalPages(Math.floor(total / limit) + 1); + setArchives({archives, total}); + } + + useEffect(() => { + search(searchStr); + }, [offset, limit, domain, searchStr]); + + return { + archivesData, + totalPages, + setTotalPages, + offset, + setOffset, + setPage, + limit, + setLimit, + domain, + setDomain, + searchStr, + setSearchStr, + activePage, + setActivePage, + search, + }; +} diff --git a/app/src/hooks/useDomains.js b/app/src/hooks/useDomains.js new file mode 100644 index 00000000..19766522 --- /dev/null +++ b/app/src/hooks/useDomains.js @@ -0,0 +1,21 @@ +import {useEffect, useState} from "react"; +import {fetchDomains} from "../api"; + +export const useDomains = () => { + const [domains, setDomains] = useState(null); + const [total, setTotal] = useState(null); + + const _fetchDomains = async () => { + setDomains(null); + setTotal(0); + let [domains, total] = await fetchDomains(); + setDomains(domains); + setTotal(total); + } + + useEffect(() => { + _fetchDomains(); + }, []); + + return [domains, total]; +} diff --git a/app/src/hooks/useSearchParam.js b/app/src/hooks/useSearchParam.js new file mode 100644 index 00000000..7d20264d --- /dev/null +++ b/app/src/hooks/useSearchParam.js @@ -0,0 +1,21 @@ +import {useEffect, useState} from "react"; +import {useHistory} from "react-router-dom"; + +export const useSearchParam = (key, defaultValue = null) => { + // Get a window.location.search param. + const startingValue = new URLSearchParams(window.location.search).get(key); + + const [value, setValue] = useState(startingValue || defaultValue); + const history = useHistory(); + + useEffect(() => { + const params = new URLSearchParams(window.location.search); + params.delete(key); + if (value) { + params.append(key, value); + } + history.push({search: params.toString()}); + }, [value]) + + return [value, setValue]; +} diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index e6cdd7aa..8d33572b 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -30,7 +30,7 @@ class ArchiveDownloader(Downloader, ABC): if download.attempts > 3: raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}') - lib.new_archive(download.url, sync=True) + lib.do_archive(download.url) return True diff --git a/modules/archive/api.py b/modules/archive/api.py index 30309d00..22643cdc 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -9,8 +9,7 @@ from wrolpi.errors import ValidationError from wrolpi.root_api import get_blueprint, json_response from wrolpi.schema import validate_doc, JSONErrorResponse from . import lib -from .schema import RetrieveURLsRequest, RetrieveURLsResponse, PostArchiveRequest, ArchiveSearchRequest, \ - ArchiveSearchResponse +from .schema import ArchiveSearchRequest, ArchiveSearchResponse NAME = 'archive' @@ -19,22 +18,9 @@ bp = get_blueprint('Archive', '/api/archive') logger = logger.getChild(__name__) -@bp.post('/') -@validate_doc( - 'Archive a website', - PostArchiveRequest, -) -@wrol_mode_check -async def post_archive(_: Request, data: dict): - # Remove whitespace from URL. - url = data['url'].strip() - lib.new_archive(url) - return response.empty() - - @bp.delete('/') @validate_doc( - 'Delete a website record', + 'Delete an individual archive', responses=[ (HTTPStatus.NOT_FOUND, JSONErrorResponse), ] @@ -45,27 +31,6 @@ async def delete_archive(_: Request, archive_id: int): return response.empty() -@bp.post('/urls') -@validate_doc( - 'Get a list of URLs', - RetrieveURLsRequest, - RetrieveURLsResponse, -) -async def get_urls(_: Request, data: dict): - try: - limit = abs(int(data.get('limit', 20))) - offset = abs(int(data.get('offset', 0))) - domain = data.get('domain') - except Exception as e: - logger.error(f'Bad request', exc_info=e) - return response.json(dict(error='bad request'), HTTPStatus.BAD_REQUEST) - - urls = lib.get_urls(limit, offset, domain) - count = lib.get_url_count(domain) - ret = dict(urls=urls, totals=dict(urls=count)) - return json_response(ret) - - @bp.post('/refresh') @wrol_mode_check async def refresh_archives(_: Request): diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py index ebd1f51c..7da2dadd 100644 --- a/modules/archive/conftest.py +++ b/modules/archive/conftest.py @@ -5,7 +5,7 @@ from uuid import uuid4 import pytest -from modules.archive.models import Archive, URL, Domain +from modules.archive.models import Archive, Domain from wrolpi.dates import strftime_ms @@ -58,32 +58,20 @@ def archive_factory(test_session, archive_directory): test_session.add(domain) test_session.flush([domain]) - url_ = test_session.query(URL).filter_by(url=url).one_or_none() - if not url_: - url_ = URL( - url=url, - domain_id=domain.id, - ) - test_session.add(url_) - test_session.flush([url_]) - archive = Archive( readability_json_path=readability_json_path, readability_path=readability_path, readability_txt_path=readability_txt_path, screenshot_path=screenshot_path, singlefile_path=singlefile_path, - status='complete', title=title, - url_id=url_.id, + url=url, domain_id=domain.id, contents=contents, archive_datetime=timestamp, ) test_session.add(archive) - test_session.flush([archive]) - url_.latest_id = archive.id test_session.commit() for path in archive.my_paths(): diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 39618236..b41d9ef7 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -1,4 +1,3 @@ -import asyncio import base64 import gzip import json @@ -11,12 +10,11 @@ import requests from bs4 import BeautifulSoup from sqlalchemy.orm import Session -from modules.archive.models import URL, Domain, Archive +from modules.archive.models import Domain, Archive from wrolpi.common import get_media_directory, logger, chunks, extract_domain from wrolpi.dates import now, strptime_ms, strftime_ms from wrolpi.db import get_db_session, get_db_curs, get_ranked_models -from wrolpi.errors import InvalidDomain, UnknownURL, PendingArchive, InvalidArchive -from wrolpi.vars import PYTEST +from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive logger = logger.getChild(__name__) @@ -109,122 +107,71 @@ def request_archive(url: str): return singlefile, readability, screenshot -def is_pending_archive(url: str) -> bool: - with get_db_session() as session: - url = session.query(URL).filter_by(url=url).one_or_none() - if url and url.latest.status == 'pending': - return True - return False - - -def new_archive(url: str, sync: bool = False): +def do_archive(url: str) -> Archive: """ - Request archiving of the provided URL. Store the returned files in their domain's directory. - - :param url: The URL to archive. - :param sync: Perform the archiving process synchronously for testing. + Perform the real archive request to the archiving service. Store the resulting data into files. Create an Archive + record in the DB. Create Domain/URL if missing. """ - # Check that the archive files are available. - get_new_archive_files(url) + singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \ + get_new_archive_files(url) - if is_pending_archive(url): - raise PendingArchive() + singlefile, readability, screenshot = request_archive(url) + + # Store the single-file HTML in its own file. + with singlefile_path.open('wt') as fh: + fh.write(singlefile) + if screenshot: + with screenshot_path.open('wb') as fh: + fh.write(screenshot) + else: + screenshot_path = None + + # Store the Readability into separate files. This allows the user to view text-only or html articles. + title = None + readability_txt = None + if readability: + title = readability.get('title') + + # Write the readability parts to their own files. Write what is left after pops to the JSON file. + with readability_path.open('wt') as fh: + fh.write(readability.pop('content')) + with readability_txt_path.open('wt') as fh: + readability_txt = readability.pop('textContent') + fh.write(readability_txt) + else: + readability_path = readability_txt_path = None + + # Always write a JSON file that contains at least the URL. + readability = readability or {} + # Use the Readability title, or try and extract one from singlefile. + if not title and singlefile: + title = get_title_from_html(singlefile, url=url) + readability['title'] = title + readability['url'] = url + with readability_json_path.open('wt') as fh: + fh.write(json.dumps(readability)) with get_db_session(commit=True) as session: - domain, url_ = get_or_create_domain_and_url(session, url) - + domain = get_or_create_domain(session, url) archive = Archive( - url_id=url_.id, - domain_id=domain.id, - status='pending', + title=title, archive_datetime=now(), + singlefile_path=singlefile_path, + readability_path=readability_path, + readability_json_path=readability_json_path, + readability_txt_path=readability_txt_path, + screenshot_path=screenshot_path, + contents=readability_txt, + url=url, + domain_id=domain.id, ) session.add(archive) session.flush() - archive_id = archive.id - - url_.latest_id = archive_id - url_.latest_datetime = now() - - if PYTEST or sync: - return _do_archive(url, archive_id) - else: - # Run the real archive process in the future. - asyncio.ensure_future(do_archive(url, archive_id)) return archive -def _do_archive(url: str, archive_id: int): - """ - Perform the real archive request to the archiving service. Store the resulting data into files. Update the Archive - in the DB. - """ - try: - singlefile, readability, screenshot = request_archive(url) - - singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \ - get_new_archive_files(url) - - # Store the single-file HTML in it's own file. - with singlefile_path.open('wt') as fh: - fh.write(singlefile) - if screenshot: - with screenshot_path.open('wb') as fh: - fh.write(screenshot) - else: - screenshot_path = None - - # Store the Readability into separate files. This allows the user to view text-only or html articles. - title = None - if readability: - title = readability.get('title') - - # Write the readability parts to their own files. Write what is left after pops to the JSON file. - with readability_path.open('wt') as fh: - fh.write(readability.pop('content')) - with readability_txt_path.open('wt') as fh: - fh.write(readability.pop('textContent')) - else: - readability_path = readability_txt_path = None - - # Always write a JSON file that contains at least the URL. - readability = readability or {} - # Use the Readability title, or try and extract one from singlefile. - if not title and singlefile: - title = get_title_from_html(singlefile, url=url) - readability['title'] = title - readability['url'] = url - with readability_json_path.open('wt') as fh: - fh.write(json.dumps(readability)) - - with get_db_session(commit=True) as session: - archive = session.query(Archive).filter_by(id=archive_id).one() - archive.status = 'complete' - archive.archive_datetime = now() - archive.title = title - archive.singlefile_path = singlefile_path - archive.readability_path = readability_path - archive.readability_json_path = readability_json_path - archive.readability_txt_path = readability_txt_path - archive.screenshot_path = screenshot_path - # Update the latest for easy viewing. - archive.url.latest_id = archive.id - archive.url.latest_datetime = archive.archive_datetime - - return archive - except Exception: - with get_db_session(commit=True) as session: - archive = session.query(Archive).filter_by(id=archive_id).one() - archive.status = 'failed' - raise - - -async def do_archive(url: str, archive_id: int): - _do_archive(url, archive_id) - - -def get_or_create_domain_and_url(session, url): +def get_or_create_domain(session, url) -> Domain: """ Get/create the Domain for this archive. """ @@ -234,12 +181,7 @@ def get_or_create_domain_and_url(session, url): domain = Domain(domain=domain_, directory=str(get_domain_directory(url))) session.add(domain) session.flush() - url_ = session.query(URL).filter_by(url=url).one_or_none() - if not url_: - url_ = URL(url=url, domain_id=domain.id) - session.add(url_) - session.flush() - return domain, url_ + return domain def get_title_from_html(html: str, url: str = None) -> str: @@ -260,42 +202,6 @@ def get_domain(session, domain: str) -> Domain: return domain_ -def get_urls(limit: int = 20, offset: int = 0, domain: str = ''): - with get_db_session() as session: - if domain: - domain_ = get_domain(session, domain) - urls = domain_.urls[offset:offset + limit] - urls = sorted(urls, key=lambda i: i.latest_datetime)[::-1] - else: - urls = session.query(URL) \ - .order_by(URL.latest_datetime.desc()) \ - .limit(limit) \ - .offset(offset) \ - .all() - urls = [i.dict() for i in urls] - return urls - - -def get_url_count(domain: str = '') -> int: - """ - Get count of all URLs. Or, get count of all attached to a specific domain string. - """ - with get_db_session() as session: - domain_id = None - if domain: - domain_id = get_domain(session, domain).id - - with get_db_curs() as curs: - stmt = 'SELECT COUNT(*) FROM url' - params = {} - if domain_id: - stmt = f'{stmt} WHERE domain_id = %(domain_id)s' - params['domain_id'] = domain_id - curs.execute(stmt, params) - count = int(curs.fetchone()[0]) - return count - - def delete_archive(archive_id: int): """ Delete an Archive and all of it's files. @@ -307,14 +213,6 @@ def delete_archive(archive_id: int): # Delete any files associated with this URL. archive.delete() - # URL must have the latest - url = archive.url - url.update_latest() - # Delete this archive. - session.query(Archive).filter_by(id=archive_id).delete() - # Remove the URL if necessary. - if not url.archives: - session.query(URL).filter_by(id=url.id).delete() def group_archive_files(files: Iterator[pathlib.Path]) -> groupby: @@ -392,35 +290,19 @@ def _refresh_archives(): if archive_count: logger.info(f'Inserted/updated {archive_count} archives') - with get_db_session(commit=True) as session: - # Check that each URLs most recent Archive exists, if it does not use the next most recent. - urls = session.query(URL).all() - for url in urls: - if not url.latest or not url.latest.singlefile_path or not url.latest.singlefile_path.path.exists(): - url.update_latest() - session.commit() - singlefile_paths = list(singlefile_paths) with get_db_curs(commit=True) as curs: curs.execute('DELETE FROM archive WHERE singlefile_path != ALL(%s)', (singlefile_paths,)) with get_db_curs(commit=True) as curs: - stmt = ''' - DELETE FROM url WHERE id NOT IN ( - select distinct url_id from archive - ) RETURNING url.id - ''' - curs.execute(stmt) - urls = list(map(dict, curs.fetchall())) stmt = ''' DELETE FROM domains WHERE - id NOT IN (select distinct domain_id from url) - AND id NOT IN (select distinct domain_id from archive) + id NOT IN (select distinct domain_id from archive) RETURNING domains.id ''' curs.execute(stmt) domains = list(map(dict, curs.fetchall())) - logger.info(f'Deleted {len(urls)} URLS and {len(domains)} Domains') + logger.info(f'Deleted {len(domains)} Domains') with get_db_session(commit=True) as session: # Get all archives that have no text contents, but have txt files. @@ -450,40 +332,33 @@ def upsert_archive(dt: str, files, session: Session): except Exception as e: raise InvalidArchive() from e - domain, url_ = get_or_create_domain_and_url(session, url) - logger.debug(f'Upsert url={url_.url}') + domain = get_or_create_domain(session, url) # Get the existing Archive, or create a new one. archive = session.query(Archive).filter_by(singlefile_path=singlefile_path).one_or_none() if not archive: # No archive matches this singlefile_path, create a new one. archive = Archive( - url_id=url_.id, + url=url, domain_id=domain.id, ) session.add(archive) session.flush() # Update the archive with the files that we have. - archive.status = 'complete', archive.archive_datetime = dt archive.singlefile_path = singlefile_path archive.readability_path = readability_path archive.readability_txt_path = readability_txt_path archive.readability_json_path = readability_json_path archive.screenshot_path = screenshot_path - archive_id = archive.id - - if not url_.latest_datetime or url_.latest_datetime < dt: - url_.latest_id = archive_id - url_.latest_datetime = archive.archive_datetime def get_domains(): with get_db_curs() as curs: stmt = ''' - SELECT domains.domain AS domain, COUNT(u.id) AS url_count + SELECT domains.domain AS domain, COUNT(a.id) AS url_count FROM domains - LEFT JOIN url u on domains.id = u.domain_id + LEFT JOIN archive a on domains.id = a.domain_id GROUP BY domains.domain ORDER BY domains.domain ''' diff --git a/modules/archive/models.py b/modules/archive/models.py index 970a9329..908afa7a 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -13,8 +13,6 @@ class Archive(Base, ModelHelper): __tablename__ = 'archive' id = Column(Integer, primary_key=True) - url_id = Column(Integer, ForeignKey('url.id', ondelete='cascade')) - url = relationship('URL', primaryjoin='Archive.url_id==URL.id') domain_id = Column(Integer, ForeignKey('domains.id')) domain = relationship('Domain', primaryjoin='Archive.domain_id==Domain.id') @@ -24,16 +22,16 @@ class Archive(Base, ModelHelper): readability_txt_path = Column(MediaPathType) screenshot_path = Column(MediaPathType) + url = Column(String) title = Column(String) archive_datetime = Column(TZDateTime) - status = Column(String) contents = Column(String) textsearch = Column(tsvector, Computed('''setweight(to_tsvector('english'::regconfig, title), 'A') || setweight(to_tsvector('english'::regconfig, contents), 'D')''')) def __repr__(self): - return f'' + return f'' def my_paths(self) -> Generator: if self.singlefile_path: @@ -65,12 +63,6 @@ class Archive(Base, ModelHelper): return self.archive_datetime > other.archive_datetime def __json__(self): - url = None - if self.url: - url = dict( - url=self.url.url, - id=self.url.id, - ) d = dict( archive_datetime=self.archive_datetime, domain=self.domain.dict() if self.domain else None, @@ -81,10 +73,8 @@ class Archive(Base, ModelHelper): readability_txt_path=self.readability_txt_path, screenshot_path=self.screenshot_path, singlefile_path=self.singlefile_path, - status=self.status, title=self.title, - url=url, - url_id=self.url_id, + url=self.url, ) return d @@ -93,63 +83,12 @@ class Archive(Base, ModelHelper): session = Session.object_session(self) - if self.url: - self.url.update_latest() - if not self.url.latest_id: - self.url.delete() - session.query(Archive).filter_by(id=self.id).delete() - -class URL(Base, ModelHelper): - __tablename__ = 'url' - id = Column(Integer, primary_key=True) - - url = Column(String, unique=True) - - archives: InstrumentedList = relationship('Archive', primaryjoin='URL.id==Archive.url_id') - latest_id = Column(Integer, ForeignKey('archive.id')) - latest = relationship('Archive', primaryjoin='URL.latest_id==Archive.id') - latest_datetime = Column(TZDateTime) - domain_id = Column(Integer, ForeignKey('domains.id')) - domain = relationship('Domain', primaryjoin='URL.domain_id==Domain.id') - - def __repr__(self): - return f'' - - def dict(self) -> dict: - d = super().dict() - d['latest'] = self.latest.dict() if self.latest else None - d['domain'] = self.domain.dict() - d['archives'] = [i.dict() for i in self.archives] - return d - - def update_latest(self): - """ - Set `latest` to the most recent Archive of this URL whose `singlefile` exists. - """ - self.latest_id = None - archives = filter(lambda i: i.singlefile_path and i.singlefile_path.path.exists(), self.archives) - try: - latest: Archive = max(archives) - if latest: - self.latest_id = latest.id - except ValueError: - # No archives exist! - pass - - def delete(self): - session = Session.object_session(self) - archive_ids = [int(i.id) for i in self.archives] - if archive_ids: - for id_ in archive_ids: - session.query(Archive).filter_by(id=id_).delete() - - domain = self.domain - session.query(URL).filter_by(id=self.id).delete() - - if len(domain.urls) == 0: - domain.delete() + if self.domain: + domain_archives = [i.id for i in self.domain.archives] + if not domain_archives: + self.domain.delete() class Domain(Base, ModelHelper): @@ -159,7 +98,7 @@ class Domain(Base, ModelHelper): domain = Column(String) directory = Column(MediaPathType) - urls: InstrumentedList = relationship('URL', primaryjoin='Domain.id==URL.domain_id', order_by='URL.latest_datetime') + archives: InstrumentedList = relationship('Archive', primaryjoin='Archive.domain_id==Domain.id') def __repr__(self): return f'' @@ -167,5 +106,4 @@ class Domain(Base, ModelHelper): def delete(self): session = Session.object_session(self) session.execute('DELETE FROM archive WHERE domain_id=:id', dict(id=self.id)) - session.execute('DELETE FROM url WHERE domain_id=:id', dict(id=self.id)) session.query(Domain).filter_by(id=self.id).delete() diff --git a/modules/archive/schema.py b/modules/archive/schema.py index 3ef79b7a..99443194 100644 --- a/modules/archive/schema.py +++ b/modules/archive/schema.py @@ -1,12 +1,6 @@ from sanic_openapi import doc -class RetrieveURLsRequest: - limit = doc.Integer() - offset = doc.Integer() - domain = doc.String() - - class ArchiveDict: id = doc.Integer() url_id = doc.Integer() @@ -25,25 +19,6 @@ class DomainDict: domain = doc.String() -class URLDict: - id = doc.Integer() - url = doc.String() - latest_id = doc.Integer() - latest_datetime = doc.DateTime() - domain_id = doc.Integer() - domain = DomainDict - latest = ArchiveDict - archives = doc.List(ArchiveDict) - - -class RetrieveURLsResponse: - urls = doc.List(URLDict) - - -class PostArchiveRequest: - url = doc.String(required=True) - - class ArchiveSearchRequest: search_str = doc.String() domain = doc.String() diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 82f0c10e..39fe6da6 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -4,12 +4,11 @@ from datetime import datetime import mock -from modules.archive.lib import new_archive, get_or_create_domain_and_url, get_urls, get_url_count, _refresh_archives, \ - get_new_archive_files, delete_archive -from modules.archive.models import Archive, URL, Domain +from modules.archive.lib import get_or_create_domain, _refresh_archives, \ + get_new_archive_files, delete_archive, do_archive, get_domains +from modules.archive.models import Archive, Domain from wrolpi.common import get_media_directory from wrolpi.db import get_db_session -from wrolpi.errors import InvalidDomain from wrolpi.media_path import MediaPath from wrolpi.root_api import CustomJSONEncoder from wrolpi.test.common import TestAPI, wrap_test_db, wrap_media_directory @@ -55,7 +54,7 @@ class TestArchive(TestAPI): @wrap_test_db def test_new_archive(self): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): - archive1 = new_archive('https://example.com', sync=True) + archive1 = do_archive('https://example.com') # Everything is filled out. self.assertIsInstance(archive1, Archive) self.assertIsNotNone(archive1.archive_datetime) @@ -77,15 +76,14 @@ class TestArchive(TestAPI): with open(archive1.readability_json_path.path) as fh: self.assertEqual(json.load(fh), {'title': 'ジにてこちら', 'url': 'https://example.com'}) - archive2 = new_archive('https://example.com') - # URL and Domain are reused. - self.assertEqual(archive1.url, archive2.url) + archive2 = do_archive('https://example.com') + # Domain is reused. self.assertEqual(archive1.domain, archive2.domain) @wrap_test_db def test_no_screenshot(self): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)): - archive = new_archive('https://example.com', sync=True) + archive = do_archive('https://example.com') self.assertIsInstance(archive.singlefile_path, MediaPath) self.assertIsInstance(archive.readability_path, MediaPath) # Screenshot was empty @@ -94,7 +92,7 @@ class TestArchive(TestAPI): @wrap_test_db def test_no_readability(self): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(readability=False)): - archive = new_archive('https://example.com', sync=True) + archive = do_archive('https://example.com') self.assertIsInstance(archive.singlefile_path, MediaPath) self.assertIsInstance(archive.screenshot_path, MediaPath) # Readability empty @@ -104,115 +102,41 @@ class TestArchive(TestAPI): @wrap_test_db def test_dict(self): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): - d = new_archive('https://example.com', sync=True).dict() + d = do_archive('https://example.com').dict() self.assertIsInstance(d, dict) json.dumps(d, cls=CustomJSONEncoder) @wrap_test_db def test_relationships(self): with get_db_session(commit=True) as session: - domain, url = get_or_create_domain_and_url(session, 'https://wrolpi.org:443') + url = 'https://wrolpi.org:443' + domain = get_or_create_domain(session, url) archive = Archive( singlefile_path=f'{self.tmp_dir.name}/wrolpi.org:443/foo', title='bar', - url_id=url.id, + url=url, domain_id=domain.id, ) session.add(archive) session.flush() - url.latest_id = archive.id - self.assertEqual(archive.domain, domain) - self.assertEqual(archive.url, url) - - # Relationships are added in the dict() method. - self.assertDictContains( - url.dict(), - dict( - id=1, - url='https://wrolpi.org:443', - latest_id=1, - latest=dict(singlefile_path=self.tmp_path / 'wrolpi.org:443/foo'), - domain_id=1, - domain=dict(directory=self.tmp_path / 'archive/wrolpi.org:443', domain='wrolpi.org:443'), - ) - ) - - @wrap_test_db - def test_get_urls(self): - # No URLs yet. - urls = get_urls() - self.assertEqual([], urls) - - with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(readability=False)): - # One set of duplicate URLs - new_archive('https://wrolpi.org/one', sync=True) - new_archive('https://wrolpi.org/one', sync=True) - - # Unique URLs - new_archive('https://wrolpi.org/two', sync=True) - new_archive('https://wrolpi.org/three', sync=True) - new_archive('https://example.com/one', sync=True) - - urls = get_urls() - # There are only 4 because one set is duplicate. - self.assertEqual(len(urls), 4) - self.assertEqual( - ['https://example.com/one', 'https://wrolpi.org/three', - 'https://wrolpi.org/two', 'https://wrolpi.org/one'], - [i['url'] for i in urls], - ) - - # Only one URL for this domain. - urls = get_urls(domain='example.com') - self.assertEqual(len(urls), 1) - self.assertEqual(urls[0]['url'], 'https://example.com/one') - - # Bad domain requested. - self.assertRaises(InvalidDomain, get_urls, domain='bad_domain.com') - - # Limit to 3, but with an offset of 2 there are only 2. - urls = get_urls(3, 2) - self.assertEqual(['https://wrolpi.org/two', 'https://wrolpi.org/one'], [i['url'] for i in urls]) - - # First two of this domain. - urls = get_urls(2, 0, 'wrolpi.org') - self.assertEqual(['https://wrolpi.org/two', 'https://wrolpi.org/one'], [i['url'] for i in urls]) - # Last two of this domain, but there is only 1. - urls = get_urls(2, 2, 'wrolpi.org') - self.assertEqual(['https://wrolpi.org/three'], [i['url'] for i in urls]) @wrap_test_db def test_validate_paths(self): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): - archive = new_archive('https://example.com') + archive = do_archive('https://example.com') try: with get_db_session(commit=True): archive.singlefile_path = 'asdf' except ValueError as e: self.assertIn('relative', str(e), f'Relative path error was not raised') - @wrap_test_db - def test_get_url_count(self): - with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): - self.assertRaises(InvalidDomain, get_url_count, 'bad domain') - - self.assertEqual(get_url_count(), 0) - new_archive('https://example.com', sync=True) - self.assertEqual(get_url_count(), 1) - new_archive('https://example.com', sync=True) - self.assertEqual(get_url_count(), 1) - new_archive('https://example.org', sync=True) - self.assertEqual(get_url_count(), 2) - new_archive('https://example.org', sync=True) - self.assertEqual(get_url_count('example.org'), 1) - @wrap_test_db def test_get_title_from_html(self): with wrap_media_directory(): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): - archive = new_archive('example.com', sync=True) + archive = do_archive('example.com') self.assertEqual(archive.title, 'ジにてこちら') def fake_request_archive(_): @@ -225,7 +149,7 @@ class TestArchive(TestAPI): return singlefile, r, s with mock.patch('modules.archive.lib.request_archive', fake_request_archive): - archive = new_archive('example.com', sync=True) + archive = do_archive('example.com') self.assertEqual(archive.title, 'some title') def fake_request_archive(_): @@ -238,16 +162,12 @@ class TestArchive(TestAPI): return singlefile, r, s with mock.patch('modules.archive.lib.request_archive', fake_request_archive): - archive = new_archive('example.com', sync=True) + archive = do_archive('example.com') self.assertIsNone(archive.title) @wrap_test_db def test_refresh_archives(self): with wrap_media_directory(): - with get_db_session() as session: - urls = session.query(URL).count() - self.assertEqual(urls, 0) - archive_directory = get_media_directory() / 'archive' archive_directory.mkdir() @@ -274,25 +194,17 @@ class TestArchive(TestAPI): # This single archive is found. _refresh_archives() with get_db_session() as session: - self.assertEqual(session.query(URL).count(), 1) self.assertEqual(session.query(Archive).count(), 1) - url: URL = session.query(URL).one() - archive: Archive = session.query(Archive).one() - self.assertEqual(url.latest, archive) - # latest_datetime is set, the timestamp in the file name is assumed to be UTC. - self.assertEqual(str(url.latest_datetime), '2021-10-05 16:20:10.346823-06:00') - # Running the refresh does not result in a new archive. _refresh_archives() with get_db_session() as session: - self.assertEqual(session.query(URL).count(), 1) self.assertEqual(session.query(Archive).count(), 1) def test_archive_refresh_deleted_archive(test_session, archive_directory, archive_factory): """ - Archives/URLs/Domains should be deleted when archive files are deleted. + Archives/Domains should be deleted when archive files are deleted. """ archive1 = archive_factory('example.com', 'https://example.com/1') archive2 = archive_factory('example.com', 'https://example.com/1') @@ -304,33 +216,32 @@ def test_archive_refresh_deleted_archive(test_session, archive_directory, archiv empty = archive_directory / 'empty' empty.mkdir() - def check_counts(archive_count, url_count, domain_count): + def check_counts(archive_count, domain_count): assert test_session.query(Archive).count() == archive_count - assert test_session.query(URL).count() == url_count assert test_session.query(Domain).count() == domain_count # All 3 archives are already in the DB. - check_counts(archive_count=5, url_count=4, domain_count=3) + check_counts(archive_count=5, domain_count=3) _refresh_archives() - check_counts(archive_count=5, url_count=4, domain_count=3) + check_counts(archive_count=5, domain_count=3) # Delete archive2's files, it's the latest for 'https://example.com/1' for path in archive2.my_paths(): path.unlink() _refresh_archives() - check_counts(archive_count=4, url_count=4, domain_count=3) + check_counts(archive_count=4, domain_count=3) # Delete archive1's files, now the URL is empty. for path in archive1.my_paths(): path.unlink() _refresh_archives() - check_counts(archive_count=3, url_count=3, domain_count=3) + check_counts(archive_count=3, domain_count=3) # Delete archive3, now there is now example.com domain for path in archive3.my_paths(): path.unlink() _refresh_archives() - check_counts(archive_count=2, url_count=2, domain_count=2) + check_counts(archive_count=2, domain_count=2) # Delete all the rest of the archives for path in archive4.my_paths(): @@ -338,7 +249,7 @@ def test_archive_refresh_deleted_archive(test_session, archive_directory, archiv for path in archive5.my_paths(): path.unlink() _refresh_archives() - check_counts(archive_count=0, url_count=0, domain_count=0) + check_counts(archive_count=0, domain_count=0) def test_refresh_archives_fills_contents(test_session, archive_factory, test_client): @@ -380,18 +291,36 @@ def test_delete_archive(test_session, archive_factory): archive2 = archive_factory('example.com', 'https://example.com/1') archive3 = archive_factory('example.com', 'https://example.com/1') - url = test_session.query(URL).one() - assert url.latest == archive3 - # Delete the oldest. delete_archive(archive1.id) # Delete the latest. delete_archive(archive3.id) - # Delete the last archive. The URL and Domain should also be deleted. + # Delete the last archive. The Domain should also be deleted. delete_archive(archive2.id) - url = test_session.query(URL).one_or_none() - assert url is None domain = test_session.query(Domain).one_or_none() assert domain is None + + +def test_get_domains(test_session, archive_factory): + """ + `get_domains` gets only Domains with Archives. + """ + archive1 = archive_factory('example.com') + archive2 = archive_factory('example.com') + archive3 = archive_factory('example.org') + + assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org'] + + archive2.delete() + test_session.commit() + assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org'] + + archive1.delete() + test_session.commit() + assert [i['domain'] for i in get_domains()] == ['example.org'] + + archive3.delete() + test_session.commit() + assert [i['domain'] for i in get_domains()] == []