Feature/archive search (#70)

* Working through archive search.

* Using custom hook.

* Setting archive.contents duration creation.

* Archive search and domain list use custom hooks.

* Watching search param using value, not history.

* Search params are preserved on load.  Search and domain can be cleared.

* Fixing archive external link.

* Archive search form is new generic SearchInput.

* Reworking Archives empty states.

* Simplifying Archives logic.

* Fixing bug which displayed the wrong empty state during loading.

* Archives can be synced and deleted.

* Changing wording of archive delete.

* Removing URL table.

* Deleting domain if it has no archives.

* Fixing FE use of archive.url.  Fixing bug with get_domains.

* Universal results are displayed on Welcome page.

* Creating ExternalLink to open a link in a new tab.g

* Increasing size of search headers.

* Adding divider to ease reading.

* Adding More button below search results.

* Disabling More buttons when there are no results.

* Ordering videos by rank so linked results are at the top.

Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
lrnselfreliance 2022-01-03 15:13:02 -07:00 • committed by Roland
parent b594f9761a
commit c23b014add
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
17 changed files with 666 additions and 904 deletions

View file

@ -0,0 +1,38 @@
"""Drop URLs.
Revision ID: 53f6780aa599
Revises: 31896cc29b71
Create Date: 2022-01-03 12:25:51.575505
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
from modules.archive.models import Archive
revision = '53f6780aa599'
down_revision = '31896cc29b71'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
urls = {i['id']: i['url'] for i in session.execute('SELECT id, url FROM url')}
archives = {i['id']: i['url_id'] for i in session.execute('SELECT id, url_id FROM archive')}
session.execute('ALTER TABLE archive ADD COLUMN url TEXT')
for archive_id, url_id in archives.items():
archive = session.query(Archive).filter_by(id=archive_id).one()
archive.url = urls[url_id]
session.execute('ALTER TABLE archive DROP COLUMN url_id')
session.execute('DROP TABLE url')

View file

@ -39,7 +39,7 @@ pre.wrap-text {
text-decoration: none !important;
}
.video-card-link {
.card-link {
color: black;
}

View file

@ -1,32 +1,83 @@
import React from 'react';
import React, {useEffect, useState} from 'react';
import './App.css';
import {NavBar} from "./components/Nav";
import {Route, Switch} from "react-router-dom";
import {FavoriteVideosPreview, VideosRoute, VideoWrapper, ViewedVideosPreview} from "./components/Videos";
import {Route, Switch, useHistory} from "react-router-dom";
import {VideosPreview, VideosRoute, VideoWrapper} from "./components/Videos";
import Admin from "./components/Admin";
import {Container, Divider, Header, Segment} from "semantic-ui-react";
import {Container, Divider, Header} from "semantic-ui-react";
import 'semantic-ui-offline/semantic.min.css';
import {SemanticToastContainer} from 'react-semantic-toasts';
import 'react-semantic-toasts/styles/react-semantic-alert.css';
import {AppsRoute} from "./components/Apps";
import {InventoryRoute} from "./components/Inventory";
import {ArchiveRoute} from "./components/Archive";
import {ArchiveRoute, ArchivesList} from "./components/Archive";
import {Saver} from "./components/Upload";
import {useSearchParam} from "./hooks/useSearchParam";
import {searchArchives, searchVideos} from "./api";
import {MoreButton, SearchInput} from "./components/Common";
const useSearch = () => {
let [searchStr, setSearchStr] = useSearchParam('q');
const [archives, setArchives] = useState();
const [videos, setVideos] = useState();
const localSearchArchives = async (term) => {
setArchives(null);
const [archives, total] = await searchArchives(0, 6, null, term);
setArchives(archives);
}
const localSearchVideos = async (term) => {
setVideos(null);
const [videos, total] = await searchVideos(0, 6, null, term);
setVideos(videos);
}
useEffect(() => {
localSearchArchives(searchStr);
localSearchVideos(searchStr)
}, [searchStr]);
return {searchStr, setSearchStr, archives, videos}
}
function Welcome() {
return (
<Container style={{marginTop: '2em'}}>
const {searchStr, setSearchStr, archives, videos} = useSearch();
const history = useHistory();
let body = (
<>
<Header as='h2'>Save your media</Header>
<Saver/>
</>
);
if (searchStr) {
let archiveMore = () => {
history.push(`/archive?q=${searchStr}`);
};
let videosMore = () => {
history.push(`/videos?q=${searchStr}&o=rank`);
};
body = (<>
<Header as='h2'>Archives</Header>
<ArchivesList archives={archives} searchStr={searchStr}/>
<MoreButton onClick={archiveMore} disabled={!archives || archives.length === 0}/>
<Divider/>
<Segment>
<FavoriteVideosPreview/>
</Segment>
<Segment>
<ViewedVideosPreview/>
</Segment>
<Header as='h2'>Videos</Header>
<VideosPreview videos={videos}/>
<MoreButton onClick={videosMore} disabled={!videos || videos.length === 0}/>
</>);
}
return (
<Container style={{marginTop: '2em'}}>
<SearchInput initValue={searchStr} onSubmit={setSearchStr} size='big'/>
{body}
</Container>
)
}

View file

@ -307,16 +307,11 @@ export async function deleteItems(itemIds) {
await apiDelete(`${API_URI}/inventory/item/${i}`);
}
export async function postArchive(url) {
let i = {url: url};
return await apiPost(`${API_URI}/archive`, i);
}
export async function deleteArchive(archive_id) {
return await apiDelete(`${API_URI}/archive/${archive_id}`);
}
export async function searchArchives(offset, limit, domain = null) {
export async function searchArchives(offset, limit, domain = null, searchStr = null) {
// Build a search query to retrieve a list of videos from the API
offset = offset || 0;
limit = limit || DEFAULT_LIMIT;
@ -324,6 +319,9 @@ export async function searchArchives(offset, limit, domain = null) {
if (domain) {
body['domain'] = domain;
}
if (searchStr) {
body['search_str'] = searchStr;
}
let response = await apiPost(`${ARCHIVES_API}/search`, body);
if (response.status === 200) {

View file

@ -1,29 +1,31 @@
import React from "react";
import {Card, Confirm, Container, Form, Header, Icon, Image, Placeholder, Tab, Table} from "semantic-ui-react";
import Paginator, {APIForm, objectToQuery, scrollToTop, uploadDate} from "./Common";
import {deleteArchive, fetchDomains, postArchive, refreshArchives, searchArchives} from "../api";
import React, {useState} from "react";
import {Card, Confirm, Container, Icon, Image, Placeholder, Tab} from "semantic-ui-react";
import Paginator, {ClearButton, ExternalLink, SearchInput, uploadDate} from "./Common";
import {deleteArchive, postDownload, refreshArchives} from "../api";
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
import {Link, NavLink} from "react-router-dom";
import * as QueryString from "query-string";
import {ArchivePlaceholder} from "./Placeholder";
import Table from "semantic-ui-react/dist/commonjs/collections/Table";
import {useArchives} from "../hooks/useArchives";
import {useDomains} from "../hooks/useDomains";
function FailedArchiveCard({url, syncURL, deleteURL}) {
function FailedArchiveCard({archive, syncArchive, deleteArchive}) {
let syncIcon = (
<Button onClick={() => syncURL(url.url)}>
<Button onClick={syncArchive}>
<Icon name='sync' size='big'/>
</Button>
);
let trashIcon = (
<Button onClick={() => deleteURL(url.id)}>
<Button onClick={deleteArchive}>
<Icon name='trash' size='big'/>
</Button>
);
let externalIcon = (
<a href={url.url} target='_blank' rel='noopener noreferrer'>
<a href={archive.url} target='_blank' rel='noopener noreferrer'>
<Icon name='sign-out' size='big'/>
</a>
);
@ -32,7 +34,7 @@ function FailedArchiveCard({url, syncURL, deleteURL}) {
<Card>
<Card.Content>
<Card.Header>
{url.url}
{archive.url}
</Card.Header>
<Card.Description>
<p>Failed!</p>
@ -45,345 +47,172 @@ function FailedArchiveCard({url, syncURL, deleteURL}) {
);
}
class ArchiveCard extends React.Component {
constructor(props) {
super(props);
this.state = {
deleteOpen: false,
syncOpen: false,
}
function ArchiveCard({archive, syncArchive, deleteArchive}) {
const [syncOpen, setSyncOpen] = useState(false);
const [deleteOpen, setDeleteOpen] = useState(false);
const localSyncArchive = async () => {
setSyncOpen(false);
await syncArchive(archive.url);
}
const localDeleteArchive = async () => {
setDeleteOpen(false);
await deleteArchive(archive.id);
}
syncURL = async () => {
await postArchive(this.props.archive.url.url);
await this.props.fetchURLs();
let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null;
let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null;
if (archive.status === 'failed') {
return <FailedArchiveCard archive={archive} syncArchive={localSyncArchive} deleteArchive={localDeleteArchive}/>;
}
deleteURL = async () => {
await deleteArchive(this.props.archive.id);
await this.props.fetchURLs();
let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null;
let readabilityIcon = <Button icon disabled><Icon name='book' size='large'/></Button>;
if (readabilityUrl) {
readabilityIcon = (
<Button icon href={readabilityUrl} target='_blank' rel='noopener noreferrer'>
<Icon name='book' size='large'/>
</Button>);
}
render() {
let archive = this.props.archive;
let syncIcon = null;
if (syncArchive) {
syncIcon = (<>
<Button icon onClick={() => setSyncOpen(true)}>
<Icon name='sync' size='large'/>
</Button>
<Confirm
open={syncOpen}
content='Download the latest version of this URL?'
confirmButton='Confirm'
onCancel={() => setSyncOpen(false)}
onConfirm={localSyncArchive}
/>
</>);
}
let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null;
let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null;
if (archive.status === 'failed') {
return <FailedArchiveCard url={archive} syncURL={this.syncURL} deleteURL={this.deleteURL}/>;
}
let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null;
let readabilityIcon = <Button icon disabled><Icon name='book' size='large'/></Button>;
if (readabilityUrl) {
readabilityIcon = (
<Button icon href={readabilityUrl} target='_blank' rel='noopener noreferrer'>
<Icon name='book' size='large'/>
</Button>);
}
let syncIcon = (
<>
<Button icon onClick={() => this.setState({syncOpen: true})}>
<Icon name='sync' size='large'/>
let deleteIcon = null;
if (deleteArchive) {
deleteIcon = (<>
<Button icon onClick={() => setDeleteOpen(true)}>
<Icon name='trash' size='large'/>
</Button>
<Confirm
open={this.state.syncOpen}
content='Download the latest version of this URL?'
confirmButton='Confirm'
onCancel={() => this.setState({syncOpen: false})}
onConfirm={this.syncURL}
open={deleteOpen}
content='Are you sure you want to delete this Archive? All files will be deleted.'
confirmButton='Delete'
onCancel={() => setDeleteOpen(false)}
onConfirm={localDeleteArchive}
/>
</>
);
let deleteIcon = (
<>
<Button icon onClick={() => this.setState({deleteOpen: true})}>
<Icon name='trash' size='large'/>
</Button>
<Confirm
open={this.state.deleteOpen}
content='Are you sure you want to delete this URL? All files will be deleted.'
confirmButton='Delete'
onCancel={() => this.setState({deleteOpen: false})}
onConfirm={this.deleteURL}
/>
</>
)
;
let externalIcon = (
<Button icon href={archive.url} target='_blank' rel='noopener noreferrer'>
<Icon name='sign-out' size='large'/>
</Button>
);
return (
<Card>
<Card.Content>
<a href={singlefileUrl} target='_blank' rel='noopener noreferrer'>
<Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>
<Card.Header>
<Container textAlign='left'>
<p>{archive.title || archive.url}</p>
</Container>
</Card.Header>
</a>
<Card.Meta>
{uploadDate(archive.archive_datetime)}
</Card.Meta>
<Card.Description>
<Container textAlign='left'>
{readabilityIcon}
{syncIcon}
{deleteIcon}
{externalIcon}
</Container>
</Card.Description>
</Card.Content>
</Card>
)
}
let externalIcon = (
<Button icon href={archive.url} target='_blank' rel='noopener noreferrer'>
<Icon name='sign-out' size='large'/>
</Button>
);
return (
<Card>
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
<Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>
</ExternalLink>
<Card.Content>
<Card.Header>
<Container textAlign='left'>
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
<p>{archive.title || archive.url}</p>
</ExternalLink>
</Container>
</Card.Header>
<Card.Meta>
{uploadDate(archive.archive_datetime)}
</Card.Meta>
<Card.Description>
<Container textAlign='left'>
{readabilityIcon}
{syncIcon}
{deleteIcon}
{externalIcon}
</Container>
</Card.Description>
</Card.Content>
</Card>
)
}
class ArchiveCards extends React.Component {
render() {
return (
<Card.Group>
{this.props.archives.map((i) => {
return <ArchiveCard key={i['id']} archive={i} fetchURLs={this.props.fetchURLs}/>
})}
</Card.Group>
)
function ArchiveCards({archives, syncArchive, deleteArchive}) {
return (
<Card.Group>
{archives.map((i) => {
return <ArchiveCard key={i['id']} archive={i} syncArchive={syncArchive} deleteArchive={deleteArchive}/>
})}
</Card.Group>
)
}
export function ArchivesList({archives, searchStr, syncArchive, localDeleteArchive}) {
if (archives === null || archives === undefined) {
// Fetching archives.
return <ArchivePlaceholder/>;
} else if (archives.length === 0 && searchStr) {
// Search with no results.
return <p>No archives found! Is your search too restrictive?</p>;
} else if (archives.length === 0) {
// No archives fetched.
return <p>No archives found! Have you archived any webpages?</p>;
} else {
// Archives fetched successfully!
return <ArchiveCards archives={archives} syncArchive={syncArchive} deleteArchive={localDeleteArchive}/>;
}
}
class Archives extends React.Component {
export function Archives() {
const {archivesData, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} =
useArchives();
const {archives} = archivesData;
constructor(props) {
super(props);
const query = QueryString.parse(this.props.location.search);
let activePage = query.page ? parseInt(query.page) : 1;
let domain = query.domain || null;
this.state = {
activePage: activePage,
limit: 20,
archives: null,
totalPages: null,
domain: domain,
};
this.fetchURLs = this.fetchURLs.bind(this);
this.clearSearch = this.clearSearch.bind(this);
this.changePage = this.changePage.bind(this);
const syncArchive = async (url) => {
await postDownload(url, 'archive');
await search();
}
async componentDidMount() {
await this.fetchURLs();
const localDeleteArchive = async (archive_id) => {
await deleteArchive(archive_id);
await search();
}
async componentDidUpdate(prevProps, prevState, snapshot) {
if (prevProps.location.search !== this.props.location.search) {
await this.fetchURLs();
}
let domainClearButton = null;
if (domain) {
domainClearButton = <ClearButton
onClick={() => setDomain(null)}
style={{marginBottom: '1em'}}
label={`Domain: ${domain}`}
/>;
}
async fetchURLs() {
this.setState({archives: null});
let offset = this.state.limit * this.state.activePage - this.state.limit;
let [archives, total] = await searchArchives(offset, this.state.limit, this.state.domain);
let totalPages = Math.round(total / this.state.limit) || 1;
this.setState({archives: archives, totalPages: totalPages});
}
clearSearch() {
this.setState({activePage: 1, domain: null}, this.changePage);
}
changePage(activePage) {
if (activePage) {
this.setState({activePage});
}
let {history, location} = this.props;
let search = {
page: activePage > 1 ? activePage : null,
domain: this.state.domain,
};
history.push({
pathname: location.pathname,
search: objectToQuery(search),
});
scrollToTop();
}
render() {
let {archives, activePage, totalPages} = this.state;
if (archives === null) {
return (<>
<Header as='h1'>Latest Archives</Header>
<ArchivePlaceholder/>
</>)
}
let pagination = null;
if (totalPages) {
pagination = (
<div style={{marginTop: '3em', textAlign: 'center'}}>
<Paginator
activePage={activePage}
changePage={this.changePage}
totalPages={totalPages}
/>
</div>
)
}
let domainButton = null;
if (this.state.domain) {
domainButton = (
<Button icon labelPosition='right' onClick={this.clearSearch} style={{marginBottom: '1em'}}>
Search: {this.state.domain}
<Icon name='close'/>
</Button>
)
}
return (
<>
<Header as='h1'>Latest Archives</Header>
{domainButton}
<ArchiveCards archives={archives} fetchURLs={this.fetchURLs}/>
{pagination}
</>
)
}
}
class Domains extends React.Component {
constructor(props) {
super(props);
this.state = {
domains: null,
total: null,
};
this.fetchDomains = this.fetchDomains.bind(this);
}
async componentDidMount() {
await this.fetchDomains();
}
async fetchDomains() {
this.setState({domains: null});
let [domains, total] = await fetchDomains();
this.setState({domains, total});
}
row(domain) {
return <Table.Row key={domain['domain']}>
<Table.Cell>
<Link to={`/archive?domain=${domain['domain']}`}>
{domain['domain']}
</Link>
</Table.Cell>
<Table.Cell>{domain['url_count']}</Table.Cell>
</Table.Row>
}
render() {
if (this.state.domains) {
return (
<>
<Header as='h1'>Domains</Header>
<Table celled>
<Table.Header>
<Table.Row>
<Table.HeaderCell>Domain</Table.HeaderCell>
<Table.HeaderCell>URLs</Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>
{this.state.domains.map(this.row)}
</Table.Body>
</Table>
</>
)
}
return (<>
<Header as='h1'>Domains</Header>
<Placeholder>
<Placeholder.Header>
<Placeholder.Line/>
<Placeholder.Line/>
</Placeholder.Header>
</Placeholder>
</>)
}
}
class ArchiveAddForm extends APIForm {
constructor(props) {
super(props);
this.archivesRef = props.archivesRef;
this.state = {
...this.state,
inputs: {
url: '',
},
errors: {},
};
this.fetchURLs = this.fetchURLs.bind(this);
}
handleSubmit = async (e) => {
e.preventDefault();
this.setLoading();
try {
let response = await postArchive(this.state.inputs.url);
if (response.status === 204) {
this.setState({inputs: {url: ''}, success: true}, this.fetchURLs);
} else {
this.setState({loading: false, success: undefined, error: true});
}
} finally {
this.clearLoading();
}
}
fetchURLs = async (e) => {
await this.archivesRef.current.fetchURLs();
}
render() {
return (
<>
<Form onSubmit={this.handleSubmit}>
<label htmlFor='url'>Archive URL</label>
<Form.Group>
<Form.Input
name='url'
placeholder='https://wrolpi.org'
value={this.state.inputs.url}
disabled={this.state.disabled}
error={this.state.error}
onChange={this.handleInputChange}
success={this.state.success}
/>
<Form.Button primary disabled={this.state.disabled}>Archive</Form.Button>
</Form.Group>
</Form>
</>
)
}
return (
<>
<SearchInput initValue={searchStr} onSubmit={setSearchStr}/>
{domainClearButton}
<ArchivesList
archives={archives}
searchStr={searchStr}
syncArchive={syncArchive}
localDeleteArchive={localDeleteArchive}
/>
<div style={{marginTop: '3em', textAlign: 'center'}}>
<Paginator
activePage={activePage}
changePage={setPage}
totalPages={totalPages}
/>
</div>
</>
)
}
class ManageArchives extends React.Component {
@ -401,6 +230,51 @@ class ManageArchives extends React.Component {
}
}
function Domains() {
const [domains] = useDomains();
const row = (domain) => {
return <Table.Row key={domain['domain']}>
<Table.Cell>
<Link to={`/archive?domain=${domain['domain']}`}>
{domain['domain']}
</Link>
</Table.Cell>
<Table.Cell>{domain['url_count']}</Table.Cell>
</Table.Row>
}
if (domains) {
return (
<>
<Table celled>
<Table.Header>
<Table.Row>
<Table.HeaderCell>Domain</Table.HeaderCell>
<Table.HeaderCell>URLs</Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>
{domains.map(row)}
</Table.Body>
</Table>
</>
)
}
return (<>
<Placeholder>
<Placeholder.Header>
<Placeholder.Line/>
<Placeholder.Line/>
</Placeholder.Header>
</Placeholder>
</>)
}
export class ArchiveRoute extends React.Component {
constructor(props) {
@ -410,7 +284,43 @@ export class ArchiveRoute extends React.Component {
this.state = {
activeIndex: this.matchPaneTo(),
}
this.archivesRef = React.createRef();
this.archivePanes = [
{
menuItem: {
as: NavLink,
content: 'Archives',
id: 'archive',
to: '/archive',
exact: true,
key: 'home',
},
render: () => <Tab.Pane><Archives/></Tab.Pane>
},
{
menuItem: {
as: NavLink,
content: 'Domains',
id: 'domains',
to: '/archive/domains',
exact: true,
key: 'domains',
},
render: () => <Tab.Pane><Domains/></Tab.Pane>
},
{
menuItem: {
as: NavLink,
content: 'Manage',
id: 'manage',
to: '/archive/manage',
exact: true,
key: 'manage',
},
render: () => <Tab.Pane><ManageArchives/></Tab.Pane>
},
];
}
componentDidUpdate(prevProps, prevState, snapshot) {
@ -424,46 +334,9 @@ export class ArchiveRoute extends React.Component {
}
render() {
const panes = [
{
menuItem: {
as: NavLink,
content: 'Archives',
id: 'archive',
to: '/archive',
exact: true,
key: 'home',
},
render: () => <Tab.Pane><Archives {...this.props} ref={this.archivesRef}/></Tab.Pane>
},
{
menuItem: {
as: NavLink,
content: 'Domains',
id: 'domains',
to: '/archive/domains',
exact: true,
key: 'domains',
},
render: () => <Tab.Pane><Domains {...this.props}/></Tab.Pane>
},
{
menuItem: {
as: NavLink,
content: 'Manage',
id: 'manage',
to: '/archive/manage',
exact: true,
key: 'manage',
},
render: () => <Tab.Pane><ManageArchives {...this.props}/></Tab.Pane>
},
];
return (
<Container style={{marginTop: '2em', marginBottom: '2em'}}>
<ArchiveAddForm archivesRef={this.archivesRef}/>
<Tab panes={panes} activeIndex={this.state.activeIndex}/>
<Tab panes={this.archivePanes} activeIndex={this.state.activeIndex} renderActiveOnly={true}/>
</Container>
)
}

View file

@ -1,6 +1,8 @@
import React from "react";
import {Card, Container, Image, Pagination, Progress} from 'semantic-ui-react';
import React, {useState} from "react";
import {Card, Container, Form, Icon, Image, Input, Pagination, Progress} from 'semantic-ui-react';
import {Link} from "react-router-dom";
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
export const API_URI = `http://${window.location.host}/api`;
export const VIDEOS_API = `${API_URI}/videos`;
@ -99,7 +101,7 @@ export function VideoCard({video}) {
let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null;
return (
<Card style={{'width': '18em', 'margin': '1em'}}>
<Card>
<Link to={video_url}>
<Image src={poster_url} wrapped style={{position: 'relative', width: '100%'}}/>
</Link>
@ -107,7 +109,7 @@ export function VideoCard({video}) {
<Card.Content>
<Card.Header>
<Container textAlign='left'>
<Link to={video_url} className="no-link-underscore video-card-link">
<Link to={video_url} className="no-link-underscore card-link">
<p>{video.title || video.stem || video.video_path}</p>
</Link>
</Container>
@ -115,7 +117,7 @@ export function VideoCard({video}) {
<Card.Description>
<Container textAlign='left'>
{channel &&
<Link to={channel_url} className="no-link-underscore video-card-link">
<Link to={channel_url} className="no-link-underscore card-link">
<b>{channel.name}</b>
</Link>}
<p>{upload_date}</p>
@ -126,17 +128,14 @@ export function VideoCard({video}) {
)
}
export class VideoCards extends React.Component {
render() {
return (
<Card.Group>
{this.props.videos.map((v) => {
return <VideoCard key={v['id']} video={v}/>
})}
</Card.Group>
)
}
export function VideoCards({videos}) {
return (
<Card.Group>
{videos.map((v) => {
return <VideoCard key={v['id']} video={v}/>
})}
</Card.Group>
)
}
export function RequiredAsterisk() {
@ -490,3 +489,88 @@ export function objectToQuery(d) {
arr = arr.filter((i) => i !== '')
return arr.join('&');
}
export function ClearButton({onClick, style, label, icon = 'close'}) {
return <Button icon
labelPosition='right'
onClick={onClick}
style={style}
>
{label}
<Icon name={icon}/>
</Button>
}
export function SearchInput({initValue, onSubmit, size}) {
let [value, setValue] = useState(initValue || '');
size = size || 'small';
const buttonClick = () => {
// Clear the input when the "clear" button is clicked, search again.
setValue('');
onSubmit(value);
}
// Button is "search" when input is dirty, otherwise it is "clear".
let button = (
<Button icon onClick={buttonClick} style={{marginLeft: '0.5em'}} type='reset' size={size}>
<Icon name='close'/>
</Button>
);
if (!initValue || initValue !== value) {
button = (
<Button icon style={{marginLeft: '0.5em'}} type='submit' disabled={value === ''} size={size}>
<Icon name='search'/>
</Button>
);
}
const localOnSubmit = (e) => {
// Send the value up when submitting.
e.preventDefault();
onSubmit(value);
}
return (
<Form onSubmit={localOnSubmit}>
<Form.Group inline style={{marginBottom: '1em'}}>
<Input
type='text'
placeholder='Search...'
onChange={(e) => setValue(e.target.value)}
value={value}
size={size}
/>
{button}
</Form.Group>
</Form>
)
}
export function ExternalLink(props) {
return (
<Link to={props.to} target='_blank' className={props.className} rel='noopener noreferrer'>
{props.children}
</Link>
)
}
export function MoreButton(props) {
return (
<Grid columns={1} textAlign='center'>
<Grid.Row>
<Grid.Column>
<Button
onClick={props.onClick}
size='big'
style={{marginTop: '1em'}}
disabled={props.disabled}
>
{props.children}
More
</Button>
</Grid.Column>
</Grid.Row>
</Grid>
)
}

View file

@ -137,62 +137,16 @@ export class VideoWrapper extends React.Component {
}
}
class VideosPreview extends React.Component {
constructor(props) {
super(props);
this.state = {
videos: null,
};
export function VideosPreview({videos}) {
let body = <VideoPlaceholder/>
if (videos && videos.length === 0) {
body = <p>No videos available.</p>
} else if (videos && videos.length > 0) {
body = <VideoCards videos={videos}/>
}
async componentDidMount() {
await this.fetchVideos();
}
render() {
let header = <h2>{this.title}</h2>;
let body = <VideoPlaceholder/>
if (this.state.videos && this.state.videos.length === 0) {
body = <p>No videos available.</p>
} else if (this.state.videos && this.state.videos.length > 0) {
body = <VideoCards videos={this.state.videos}/>
}
return <>
{header}
{body}
</>
}
}
export class FavoriteVideosPreview extends VideosPreview {
constructor(props) {
super(props);
this.title = 'Favorite Videos'
}
async fetchVideos() {
let [videos] = await searchVideos(
0, 3, null, null, '-favorite', ['favorite']);
this.setState({videos});
}
}
export class ViewedVideosPreview extends VideosPreview {
constructor(props) {
super(props);
this.title = 'Recently Viewed Videos'
}
async fetchVideos() {
let [videos] = await searchVideos(
0, 3, null, null, '-viewed');
this.setState({videos});
}
return <>
{body}
</>
}
function FilterModal(props) {

View file

@ -0,0 +1,52 @@
import {useEffect, useState} from "react";
import {searchArchives} from "../api";
import {useSearchParam} from "./useSearchParam";
export const useArchives = (defaultLimit = 20) => {
const [archivesData, setArchives] = useState({archives: null, total: 0});
const [totalPages, setTotalPages] = useState(0);
const [activePage, setActivePage] = useState(1);
let [offset, setOffset] = useSearchParam('o');
let [limit, setLimit] = useSearchParam('l', defaultLimit);
let [domain, setDomain] = useSearchParam('domain');
let [searchStr, setSearchStr] = useSearchParam('q');
const setPage = (i) => {
i = parseInt(i);
let l = parseInt(limit);
setOffset((l * i) - l);
setActivePage(i);
}
const search = async (term) => {
setArchives({archives: null, total: 0});
setTotalPages(0);
const [archives, total] = await searchArchives(offset, limit, domain, term);
setTotalPages(Math.floor(total / limit) + 1);
setArchives({archives, total});
}
useEffect(() => {
search(searchStr);
}, [offset, limit, domain, searchStr]);
return {
archivesData,
totalPages,
setTotalPages,
offset,
setOffset,
setPage,
limit,
setLimit,
domain,
setDomain,
searchStr,
setSearchStr,
activePage,
setActivePage,
search,
};
}

View file

@ -0,0 +1,21 @@
import {useEffect, useState} from "react";
import {fetchDomains} from "../api";
export const useDomains = () => {
const [domains, setDomains] = useState(null);
const [total, setTotal] = useState(null);
const _fetchDomains = async () => {
setDomains(null);
setTotal(0);
let [domains, total] = await fetchDomains();
setDomains(domains);
setTotal(total);
}
useEffect(() => {
_fetchDomains();
}, []);
return [domains, total];
}

View file

@ -0,0 +1,21 @@
import {useEffect, useState} from "react";
import {useHistory} from "react-router-dom";
export const useSearchParam = (key, defaultValue = null) => {
// Get a window.location.search param.
const startingValue = new URLSearchParams(window.location.search).get(key);
const [value, setValue] = useState(startingValue || defaultValue);
const history = useHistory();
useEffect(() => {
const params = new URLSearchParams(window.location.search);
params.delete(key);
if (value) {
params.append(key, value);
}
history.push({search: params.toString()});
}, [value])
return [value, setValue];
}

View file

@ -30,7 +30,7 @@ class ArchiveDownloader(Downloader, ABC):
if download.attempts > 3:
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
lib.new_archive(download.url, sync=True)
lib.do_archive(download.url)
return True

View file

@ -9,8 +9,7 @@ from wrolpi.errors import ValidationError
from wrolpi.root_api import get_blueprint, json_response
from wrolpi.schema import validate_doc, JSONErrorResponse
from . import lib
from .schema import RetrieveURLsRequest, RetrieveURLsResponse, PostArchiveRequest, ArchiveSearchRequest, \
ArchiveSearchResponse
from .schema import ArchiveSearchRequest, ArchiveSearchResponse
NAME = 'archive'
@ -19,22 +18,9 @@ bp = get_blueprint('Archive', '/api/archive')
logger = logger.getChild(__name__)
@bp.post('/')
@validate_doc(
'Archive a website',
PostArchiveRequest,
)
@wrol_mode_check
async def post_archive(_: Request, data: dict):
# Remove whitespace from URL.
url = data['url'].strip()
lib.new_archive(url)
return response.empty()
@bp.delete('/<archive_id:int>')
@validate_doc(
'Delete a website record',
'Delete an individual archive',
responses=[
(HTTPStatus.NOT_FOUND, JSONErrorResponse),
]
@ -45,27 +31,6 @@ async def delete_archive(_: Request, archive_id: int):
return response.empty()
@bp.post('/urls')
@validate_doc(
'Get a list of URLs',
RetrieveURLsRequest,
RetrieveURLsResponse,
)
async def get_urls(_: Request, data: dict):
try:
limit = abs(int(data.get('limit', 20)))
offset = abs(int(data.get('offset', 0)))
domain = data.get('domain')
except Exception as e:
logger.error(f'Bad request', exc_info=e)
return response.json(dict(error='bad request'), HTTPStatus.BAD_REQUEST)
urls = lib.get_urls(limit, offset, domain)
count = lib.get_url_count(domain)
ret = dict(urls=urls, totals=dict(urls=count))
return json_response(ret)
@bp.post('/refresh')
@wrol_mode_check
async def refresh_archives(_: Request):

View file

@ -5,7 +5,7 @@ from uuid import uuid4
import pytest
from modules.archive.models import Archive, URL, Domain
from modules.archive.models import Archive, Domain
from wrolpi.dates import strftime_ms
@ -58,32 +58,20 @@ def archive_factory(test_session, archive_directory):
test_session.add(domain)
test_session.flush([domain])
url_ = test_session.query(URL).filter_by(url=url).one_or_none()
if not url_:
url_ = URL(
url=url,
domain_id=domain.id,
)
test_session.add(url_)
test_session.flush([url_])
archive = Archive(
readability_json_path=readability_json_path,
readability_path=readability_path,
readability_txt_path=readability_txt_path,
screenshot_path=screenshot_path,
singlefile_path=singlefile_path,
status='complete',
title=title,
url_id=url_.id,
url=url,
domain_id=domain.id,
contents=contents,
archive_datetime=timestamp,
)
test_session.add(archive)
test_session.flush([archive])
url_.latest_id = archive.id
test_session.commit()
for path in archive.my_paths():

View file

@ -1,4 +1,3 @@
import asyncio
import base64
import gzip
import json
@ -11,12 +10,11 @@ import requests
from bs4 import BeautifulSoup
from sqlalchemy.orm import Session
from modules.archive.models import URL, Domain, Archive
from modules.archive.models import Domain, Archive
from wrolpi.common import get_media_directory, logger, chunks, extract_domain
from wrolpi.dates import now, strptime_ms, strftime_ms
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
from wrolpi.errors import InvalidDomain, UnknownURL, PendingArchive, InvalidArchive
from wrolpi.vars import PYTEST
from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
logger = logger.getChild(__name__)
@ -109,122 +107,71 @@ def request_archive(url: str):
return singlefile, readability, screenshot
def is_pending_archive(url: str) -> bool:
with get_db_session() as session:
url = session.query(URL).filter_by(url=url).one_or_none()
if url and url.latest.status == 'pending':
return True
return False
def new_archive(url: str, sync: bool = False):
def do_archive(url: str) -> Archive:
"""
Request archiving of the provided URL. Store the returned files in their domain's directory.
:param url: The URL to archive.
:param sync: Perform the archiving process synchronously for testing.
Perform the real archive request to the archiving service. Store the resulting data into files. Create an Archive
record in the DB. Create Domain/URL if missing.
"""
# Check that the archive files are available.
get_new_archive_files(url)
singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
get_new_archive_files(url)
if is_pending_archive(url):
raise PendingArchive()
singlefile, readability, screenshot = request_archive(url)
# Store the single-file HTML in its own file.
with singlefile_path.open('wt') as fh:
fh.write(singlefile)
if screenshot:
with screenshot_path.open('wb') as fh:
fh.write(screenshot)
else:
screenshot_path = None
# Store the Readability into separate files. This allows the user to view text-only or html articles.
title = None
readability_txt = None
if readability:
title = readability.get('title')
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
with readability_path.open('wt') as fh:
fh.write(readability.pop('content'))
with readability_txt_path.open('wt') as fh:
readability_txt = readability.pop('textContent')
fh.write(readability_txt)
else:
readability_path = readability_txt_path = None
# Always write a JSON file that contains at least the URL.
readability = readability or {}
# Use the Readability title, or try and extract one from singlefile.
if not title and singlefile:
title = get_title_from_html(singlefile, url=url)
readability['title'] = title
readability['url'] = url
with readability_json_path.open('wt') as fh:
fh.write(json.dumps(readability))
with get_db_session(commit=True) as session:
domain, url_ = get_or_create_domain_and_url(session, url)
domain = get_or_create_domain(session, url)
archive = Archive(
url_id=url_.id,
domain_id=domain.id,
status='pending',
title=title,
archive_datetime=now(),
singlefile_path=singlefile_path,
readability_path=readability_path,
readability_json_path=readability_json_path,
readability_txt_path=readability_txt_path,
screenshot_path=screenshot_path,
contents=readability_txt,
url=url,
domain_id=domain.id,
)
session.add(archive)
session.flush()
archive_id = archive.id
url_.latest_id = archive_id
url_.latest_datetime = now()
if PYTEST or sync:
return _do_archive(url, archive_id)
else:
# Run the real archive process in the future.
asyncio.ensure_future(do_archive(url, archive_id))
return archive
def _do_archive(url: str, archive_id: int):
"""
Perform the real archive request to the archiving service. Store the resulting data into files. Update the Archive
in the DB.
"""
try:
singlefile, readability, screenshot = request_archive(url)
singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
get_new_archive_files(url)
# Store the single-file HTML in it's own file.
with singlefile_path.open('wt') as fh:
fh.write(singlefile)
if screenshot:
with screenshot_path.open('wb') as fh:
fh.write(screenshot)
else:
screenshot_path = None
# Store the Readability into separate files. This allows the user to view text-only or html articles.
title = None
if readability:
title = readability.get('title')
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
with readability_path.open('wt') as fh:
fh.write(readability.pop('content'))
with readability_txt_path.open('wt') as fh:
fh.write(readability.pop('textContent'))
else:
readability_path = readability_txt_path = None
# Always write a JSON file that contains at least the URL.
readability = readability or {}
# Use the Readability title, or try and extract one from singlefile.
if not title and singlefile:
title = get_title_from_html(singlefile, url=url)
readability['title'] = title
readability['url'] = url
with readability_json_path.open('wt') as fh:
fh.write(json.dumps(readability))
with get_db_session(commit=True) as session:
archive = session.query(Archive).filter_by(id=archive_id).one()
archive.status = 'complete'
archive.archive_datetime = now()
archive.title = title
archive.singlefile_path = singlefile_path
archive.readability_path = readability_path
archive.readability_json_path = readability_json_path
archive.readability_txt_path = readability_txt_path
archive.screenshot_path = screenshot_path
# Update the latest for easy viewing.
archive.url.latest_id = archive.id
archive.url.latest_datetime = archive.archive_datetime
return archive
except Exception:
with get_db_session(commit=True) as session:
archive = session.query(Archive).filter_by(id=archive_id).one()
archive.status = 'failed'
raise
async def do_archive(url: str, archive_id: int):
_do_archive(url, archive_id)
def get_or_create_domain_and_url(session, url):
def get_or_create_domain(session, url) -> Domain:
"""
Get/create the Domain for this archive.
"""
@ -234,12 +181,7 @@ def get_or_create_domain_and_url(session, url):
domain = Domain(domain=domain_, directory=str(get_domain_directory(url)))
session.add(domain)
session.flush()
url_ = session.query(URL).filter_by(url=url).one_or_none()
if not url_:
url_ = URL(url=url, domain_id=domain.id)
session.add(url_)
session.flush()
return domain, url_
return domain
def get_title_from_html(html: str, url: str = None) -> str:
@ -260,42 +202,6 @@ def get_domain(session, domain: str) -> Domain:
return domain_
def get_urls(limit: int = 20, offset: int = 0, domain: str = ''):
with get_db_session() as session:
if domain:
domain_ = get_domain(session, domain)
urls = domain_.urls[offset:offset + limit]
urls = sorted(urls, key=lambda i: i.latest_datetime)[::-1]
else:
urls = session.query(URL) \
.order_by(URL.latest_datetime.desc()) \
.limit(limit) \
.offset(offset) \
.all()
urls = [i.dict() for i in urls]
return urls
def get_url_count(domain: str = '') -> int:
"""
Get count of all URLs. Or, get count of all attached to a specific domain string.
"""
with get_db_session() as session:
domain_id = None
if domain:
domain_id = get_domain(session, domain).id
with get_db_curs() as curs:
stmt = 'SELECT COUNT(*) FROM url'
params = {}
if domain_id:
stmt = f'{stmt} WHERE domain_id = %(domain_id)s'
params['domain_id'] = domain_id
curs.execute(stmt, params)
count = int(curs.fetchone()[0])
return count
def delete_archive(archive_id: int):
"""
Delete an Archive and all of it's files.
@ -307,14 +213,6 @@ def delete_archive(archive_id: int):
# Delete any files associated with this URL.
archive.delete()
# URL must have the latest
url = archive.url
url.update_latest()
# Delete this archive.
session.query(Archive).filter_by(id=archive_id).delete()
# Remove the URL if necessary.
if not url.archives:
session.query(URL).filter_by(id=url.id).delete()
def group_archive_files(files: Iterator[pathlib.Path]) -> groupby:
@ -392,35 +290,19 @@ def _refresh_archives():
if archive_count:
logger.info(f'Inserted/updated {archive_count} archives')
with get_db_session(commit=True) as session:
# Check that each URLs most recent Archive exists, if it does not use the next most recent.
urls = session.query(URL).all()
for url in urls:
if not url.latest or not url.latest.singlefile_path or not url.latest.singlefile_path.path.exists():
url.update_latest()
session.commit()
singlefile_paths = list(singlefile_paths)
with get_db_curs(commit=True) as curs:
curs.execute('DELETE FROM archive WHERE singlefile_path != ALL(%s)', (singlefile_paths,))
with get_db_curs(commit=True) as curs:
stmt = '''
DELETE FROM url WHERE id NOT IN (
select distinct url_id from archive
) RETURNING url.id
'''
curs.execute(stmt)
urls = list(map(dict, curs.fetchall()))
stmt = '''
DELETE FROM domains WHERE
id NOT IN (select distinct domain_id from url)
AND id NOT IN (select distinct domain_id from archive)
id NOT IN (select distinct domain_id from archive)
RETURNING domains.id
'''
curs.execute(stmt)
domains = list(map(dict, curs.fetchall()))
logger.info(f'Deleted {len(urls)} URLS and {len(domains)} Domains')
logger.info(f'Deleted {len(domains)} Domains')
with get_db_session(commit=True) as session:
# Get all archives that have no text contents, but have txt files.
@ -450,40 +332,33 @@ def upsert_archive(dt: str, files, session: Session):
except Exception as e:
raise InvalidArchive() from e
domain, url_ = get_or_create_domain_and_url(session, url)
logger.debug(f'Upsert url={url_.url}')
domain = get_or_create_domain(session, url)
# Get the existing Archive, or create a new one.
archive = session.query(Archive).filter_by(singlefile_path=singlefile_path).one_or_none()
if not archive:
# No archive matches this singlefile_path, create a new one.
archive = Archive(
url_id=url_.id,
url=url,
domain_id=domain.id,
)
session.add(archive)
session.flush()
# Update the archive with the files that we have.
archive.status = 'complete',
archive.archive_datetime = dt
archive.singlefile_path = singlefile_path
archive.readability_path = readability_path
archive.readability_txt_path = readability_txt_path
archive.readability_json_path = readability_json_path
archive.screenshot_path = screenshot_path
archive_id = archive.id
if not url_.latest_datetime or url_.latest_datetime < dt:
url_.latest_id = archive_id
url_.latest_datetime = archive.archive_datetime
def get_domains():
with get_db_curs() as curs:
stmt = '''
SELECT domains.domain AS domain, COUNT(u.id) AS url_count
SELECT domains.domain AS domain, COUNT(a.id) AS url_count
FROM domains
LEFT JOIN url u on domains.id = u.domain_id
LEFT JOIN archive a on domains.id = a.domain_id
GROUP BY domains.domain
ORDER BY domains.domain
'''

View file

@ -13,8 +13,6 @@ class Archive(Base, ModelHelper):
__tablename__ = 'archive'
id = Column(Integer, primary_key=True)
url_id = Column(Integer, ForeignKey('url.id', ondelete='cascade'))
url = relationship('URL', primaryjoin='Archive.url_id==URL.id')
domain_id = Column(Integer, ForeignKey('domains.id'))
domain = relationship('Domain', primaryjoin='Archive.domain_id==Domain.id')
@ -24,16 +22,16 @@ class Archive(Base, ModelHelper):
readability_txt_path = Column(MediaPathType)
screenshot_path = Column(MediaPathType)
url = Column(String)
title = Column(String)
archive_datetime = Column(TZDateTime)
status = Column(String)
contents = Column(String)
textsearch = Column(tsvector, Computed('''setweight(to_tsvector('english'::regconfig, title), 'A') ||
setweight(to_tsvector('english'::regconfig, contents), 'D')'''))
def __repr__(self):
return f'<Archive id={self.id} url_id={self.url_id} singlefile={self.singlefile_path}>'
return f'<Archive id={self.id} url={self.url} singlefile={self.singlefile_path}>'
def my_paths(self) -> Generator:
if self.singlefile_path:
@ -65,12 +63,6 @@ class Archive(Base, ModelHelper):
return self.archive_datetime > other.archive_datetime
def __json__(self):
url = None
if self.url:
url = dict(
url=self.url.url,
id=self.url.id,
)
d = dict(
archive_datetime=self.archive_datetime,
domain=self.domain.dict() if self.domain else None,
@ -81,10 +73,8 @@ class Archive(Base, ModelHelper):
readability_txt_path=self.readability_txt_path,
screenshot_path=self.screenshot_path,
singlefile_path=self.singlefile_path,
status=self.status,
title=self.title,
url=url,
url_id=self.url_id,
url=self.url,
)
return d
@ -93,63 +83,12 @@ class Archive(Base, ModelHelper):
session = Session.object_session(self)
if self.url:
self.url.update_latest()
if not self.url.latest_id:
self.url.delete()
session.query(Archive).filter_by(id=self.id).delete()
class URL(Base, ModelHelper):
__tablename__ = 'url'
id = Column(Integer, primary_key=True)
url = Column(String, unique=True)
archives: InstrumentedList = relationship('Archive', primaryjoin='URL.id==Archive.url_id')
latest_id = Column(Integer, ForeignKey('archive.id'))
latest = relationship('Archive', primaryjoin='URL.latest_id==Archive.id')
latest_datetime = Column(TZDateTime)
domain_id = Column(Integer, ForeignKey('domains.id'))
domain = relationship('Domain', primaryjoin='URL.domain_id==Domain.id')
def __repr__(self):
return f'<URL id={self.id} url={self.url}>'
def dict(self) -> dict:
d = super().dict()
d['latest'] = self.latest.dict() if self.latest else None
d['domain'] = self.domain.dict()
d['archives'] = [i.dict() for i in self.archives]
return d
def update_latest(self):
"""
Set `latest` to the most recent Archive of this URL whose `singlefile` exists.
"""
self.latest_id = None
archives = filter(lambda i: i.singlefile_path and i.singlefile_path.path.exists(), self.archives)
try:
latest: Archive = max(archives)
if latest:
self.latest_id = latest.id
except ValueError:
# No archives exist!
pass
def delete(self):
session = Session.object_session(self)
archive_ids = [int(i.id) for i in self.archives]
if archive_ids:
for id_ in archive_ids:
session.query(Archive).filter_by(id=id_).delete()
domain = self.domain
session.query(URL).filter_by(id=self.id).delete()
if len(domain.urls) == 0:
domain.delete()
if self.domain:
domain_archives = [i.id for i in self.domain.archives]
if not domain_archives:
self.domain.delete()
class Domain(Base, ModelHelper):
@ -159,7 +98,7 @@ class Domain(Base, ModelHelper):
domain = Column(String)
directory = Column(MediaPathType)
urls: InstrumentedList = relationship('URL', primaryjoin='Domain.id==URL.domain_id', order_by='URL.latest_datetime')
archives: InstrumentedList = relationship('Archive', primaryjoin='Archive.domain_id==Domain.id')
def __repr__(self):
return f'<Domain id={self.id} domain={self.domain} directory={self.directory}>'
@ -167,5 +106,4 @@ class Domain(Base, ModelHelper):
def delete(self):
session = Session.object_session(self)
session.execute('DELETE FROM archive WHERE domain_id=:id', dict(id=self.id))
session.execute('DELETE FROM url WHERE domain_id=:id', dict(id=self.id))
session.query(Domain).filter_by(id=self.id).delete()

View file

@ -1,12 +1,6 @@
from sanic_openapi import doc
class RetrieveURLsRequest:
limit = doc.Integer()
offset = doc.Integer()
domain = doc.String()
class ArchiveDict:
id = doc.Integer()
url_id = doc.Integer()
@ -25,25 +19,6 @@ class DomainDict:
domain = doc.String()
class URLDict:
id = doc.Integer()
url = doc.String()
latest_id = doc.Integer()
latest_datetime = doc.DateTime()
domain_id = doc.Integer()
domain = DomainDict
latest = ArchiveDict
archives = doc.List(ArchiveDict)
class RetrieveURLsResponse:
urls = doc.List(URLDict)
class PostArchiveRequest:
url = doc.String(required=True)
class ArchiveSearchRequest:
search_str = doc.String()
domain = doc.String()

View file

@ -4,12 +4,11 @@ from datetime import datetime
import mock
from modules.archive.lib import new_archive, get_or_create_domain_and_url, get_urls, get_url_count, _refresh_archives, \
get_new_archive_files, delete_archive
from modules.archive.models import Archive, URL, Domain
from modules.archive.lib import get_or_create_domain, _refresh_archives, \
get_new_archive_files, delete_archive, do_archive, get_domains
from modules.archive.models import Archive, Domain
from wrolpi.common import get_media_directory
from wrolpi.db import get_db_session
from wrolpi.errors import InvalidDomain
from wrolpi.media_path import MediaPath
from wrolpi.root_api import CustomJSONEncoder
from wrolpi.test.common import TestAPI, wrap_test_db, wrap_media_directory
@ -55,7 +54,7 @@ class TestArchive(TestAPI):
@wrap_test_db
def test_new_archive(self):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
archive1 = new_archive('https://example.com', sync=True)
archive1 = do_archive('https://example.com')
# Everything is filled out.
self.assertIsInstance(archive1, Archive)
self.assertIsNotNone(archive1.archive_datetime)
@ -77,15 +76,14 @@ class TestArchive(TestAPI):
with open(archive1.readability_json_path.path) as fh:
self.assertEqual(json.load(fh), {'title': 'ジにてこちら', 'url': 'https://example.com'})
archive2 = new_archive('https://example.com')
# URL and Domain are reused.
self.assertEqual(archive1.url, archive2.url)
archive2 = do_archive('https://example.com')
# Domain is reused.
self.assertEqual(archive1.domain, archive2.domain)
@wrap_test_db
def test_no_screenshot(self):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)):
archive = new_archive('https://example.com', sync=True)
archive = do_archive('https://example.com')
self.assertIsInstance(archive.singlefile_path, MediaPath)
self.assertIsInstance(archive.readability_path, MediaPath)
# Screenshot was empty
@ -94,7 +92,7 @@ class TestArchive(TestAPI):
@wrap_test_db
def test_no_readability(self):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(readability=False)):
archive = new_archive('https://example.com', sync=True)
archive = do_archive('https://example.com')
self.assertIsInstance(archive.singlefile_path, MediaPath)
self.assertIsInstance(archive.screenshot_path, MediaPath)
# Readability empty
@ -104,115 +102,41 @@ class TestArchive(TestAPI):
@wrap_test_db
def test_dict(self):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
d = new_archive('https://example.com', sync=True).dict()
d = do_archive('https://example.com').dict()
self.assertIsInstance(d, dict)
json.dumps(d, cls=CustomJSONEncoder)
@wrap_test_db
def test_relationships(self):
with get_db_session(commit=True) as session:
domain, url = get_or_create_domain_and_url(session, 'https://wrolpi.org:443')
url = 'https://wrolpi.org:443'
domain = get_or_create_domain(session, url)
archive = Archive(
singlefile_path=f'{self.tmp_dir.name}/wrolpi.org:443/foo',
title='bar',
url_id=url.id,
url=url,
domain_id=domain.id,
)
session.add(archive)
session.flush()
url.latest_id = archive.id
self.assertEqual(archive.domain, domain)
self.assertEqual(archive.url, url)
# Relationships are added in the dict() method.
self.assertDictContains(
url.dict(),
dict(
id=1,
url='https://wrolpi.org:443',
latest_id=1,
latest=dict(singlefile_path=self.tmp_path / 'wrolpi.org:443/foo'),
domain_id=1,
domain=dict(directory=self.tmp_path / 'archive/wrolpi.org:443', domain='wrolpi.org:443'),
)
)
@wrap_test_db
def test_get_urls(self):
# No URLs yet.
urls = get_urls()
self.assertEqual([], urls)
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(readability=False)):
# One set of duplicate URLs
new_archive('https://wrolpi.org/one', sync=True)
new_archive('https://wrolpi.org/one', sync=True)
# Unique URLs
new_archive('https://wrolpi.org/two', sync=True)
new_archive('https://wrolpi.org/three', sync=True)
new_archive('https://example.com/one', sync=True)
urls = get_urls()
# There are only 4 because one set is duplicate.
self.assertEqual(len(urls), 4)
self.assertEqual(
['https://example.com/one', 'https://wrolpi.org/three',
'https://wrolpi.org/two', 'https://wrolpi.org/one'],
[i['url'] for i in urls],
)
# Only one URL for this domain.
urls = get_urls(domain='example.com')
self.assertEqual(len(urls), 1)
self.assertEqual(urls[0]['url'], 'https://example.com/one')
# Bad domain requested.
self.assertRaises(InvalidDomain, get_urls, domain='bad_domain.com')
# Limit to 3, but with an offset of 2 there are only 2.
urls = get_urls(3, 2)
self.assertEqual(['https://wrolpi.org/two', 'https://wrolpi.org/one'], [i['url'] for i in urls])
# First two of this domain.
urls = get_urls(2, 0, 'wrolpi.org')
self.assertEqual(['https://wrolpi.org/two', 'https://wrolpi.org/one'], [i['url'] for i in urls])
# Last two of this domain, but there is only 1.
urls = get_urls(2, 2, 'wrolpi.org')
self.assertEqual(['https://wrolpi.org/three'], [i['url'] for i in urls])
@wrap_test_db
def test_validate_paths(self):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
archive = new_archive('https://example.com')
archive = do_archive('https://example.com')
try:
with get_db_session(commit=True):
archive.singlefile_path = 'asdf'
except ValueError as e:
self.assertIn('relative', str(e), f'Relative path error was not raised')
@wrap_test_db
def test_get_url_count(self):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
self.assertRaises(InvalidDomain, get_url_count, 'bad domain')
self.assertEqual(get_url_count(), 0)
new_archive('https://example.com', sync=True)
self.assertEqual(get_url_count(), 1)
new_archive('https://example.com', sync=True)
self.assertEqual(get_url_count(), 1)
new_archive('https://example.org', sync=True)
self.assertEqual(get_url_count(), 2)
new_archive('https://example.org', sync=True)
self.assertEqual(get_url_count('example.org'), 1)
@wrap_test_db
def test_get_title_from_html(self):
with wrap_media_directory():
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
archive = new_archive('example.com', sync=True)
archive = do_archive('example.com')
self.assertEqual(archive.title, 'ジにてこちら')
def fake_request_archive(_):
@ -225,7 +149,7 @@ class TestArchive(TestAPI):
return singlefile, r, s
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
archive = new_archive('example.com', sync=True)
archive = do_archive('example.com')
self.assertEqual(archive.title, 'some title')
def fake_request_archive(_):
@ -238,16 +162,12 @@ class TestArchive(TestAPI):
return singlefile, r, s
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
archive = new_archive('example.com', sync=True)
archive = do_archive('example.com')
self.assertIsNone(archive.title)
@wrap_test_db
def test_refresh_archives(self):
with wrap_media_directory():
with get_db_session() as session:
urls = session.query(URL).count()
self.assertEqual(urls, 0)
archive_directory = get_media_directory() / 'archive'
archive_directory.mkdir()
@ -274,25 +194,17 @@ class TestArchive(TestAPI):
# This single archive is found.
_refresh_archives()
with get_db_session() as session:
self.assertEqual(session.query(URL).count(), 1)
self.assertEqual(session.query(Archive).count(), 1)
url: URL = session.query(URL).one()
archive: Archive = session.query(Archive).one()
self.assertEqual(url.latest, archive)
# latest_datetime is set, the timestamp in the file name is assumed to be UTC.
self.assertEqual(str(url.latest_datetime), '2021-10-05 16:20:10.346823-06:00')
# Running the refresh does not result in a new archive.
_refresh_archives()
with get_db_session() as session:
self.assertEqual(session.query(URL).count(), 1)
self.assertEqual(session.query(Archive).count(), 1)
def test_archive_refresh_deleted_archive(test_session, archive_directory, archive_factory):
"""
Archives/URLs/Domains should be deleted when archive files are deleted.
Archives/Domains should be deleted when archive files are deleted.
"""
archive1 = archive_factory('example.com', 'https://example.com/1')
archive2 = archive_factory('example.com', 'https://example.com/1')
@ -304,33 +216,32 @@ def test_archive_refresh_deleted_archive(test_session, archive_directory, archiv
empty = archive_directory / 'empty'
empty.mkdir()
def check_counts(archive_count, url_count, domain_count):
def check_counts(archive_count, domain_count):
assert test_session.query(Archive).count() == archive_count
assert test_session.query(URL).count() == url_count
assert test_session.query(Domain).count() == domain_count
# All 3 archives are already in the DB.
check_counts(archive_count=5, url_count=4, domain_count=3)
check_counts(archive_count=5, domain_count=3)
_refresh_archives()
check_counts(archive_count=5, url_count=4, domain_count=3)
check_counts(archive_count=5, domain_count=3)
# Delete archive2's files, it's the latest for 'https://example.com/1'
for path in archive2.my_paths():
path.unlink()
_refresh_archives()
check_counts(archive_count=4, url_count=4, domain_count=3)
check_counts(archive_count=4, domain_count=3)
# Delete archive1's files, now the URL is empty.
for path in archive1.my_paths():
path.unlink()
_refresh_archives()
check_counts(archive_count=3, url_count=3, domain_count=3)
check_counts(archive_count=3, domain_count=3)
# Delete archive3, now there is now example.com domain
for path in archive3.my_paths():
path.unlink()
_refresh_archives()
check_counts(archive_count=2, url_count=2, domain_count=2)
check_counts(archive_count=2, domain_count=2)
# Delete all the rest of the archives
for path in archive4.my_paths():
@ -338,7 +249,7 @@ def test_archive_refresh_deleted_archive(test_session, archive_directory, archiv
for path in archive5.my_paths():
path.unlink()
_refresh_archives()
check_counts(archive_count=0, url_count=0, domain_count=0)
check_counts(archive_count=0, domain_count=0)
def test_refresh_archives_fills_contents(test_session, archive_factory, test_client):
@ -380,18 +291,36 @@ def test_delete_archive(test_session, archive_factory):
archive2 = archive_factory('example.com', 'https://example.com/1')
archive3 = archive_factory('example.com', 'https://example.com/1')
url = test_session.query(URL).one()
assert url.latest == archive3
# Delete the oldest.
delete_archive(archive1.id)
# Delete the latest.
delete_archive(archive3.id)
# Delete the last archive. The URL and Domain should also be deleted.
# Delete the last archive. The Domain should also be deleted.
delete_archive(archive2.id)
url = test_session.query(URL).one_or_none()
assert url is None
domain = test_session.query(Domain).one_or_none()
assert domain is None
def test_get_domains(test_session, archive_factory):
"""
`get_domains` gets only Domains with Archives.
"""
archive1 = archive_factory('example.com')
archive2 = archive_factory('example.com')
archive3 = archive_factory('example.org')
assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org']
archive2.delete()
test_session.commit()
assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org']
archive1.delete()
test_session.commit()
assert [i['domain'] for i in get_domains()] == ['example.org']
archive3.delete()
test_session.commit()
assert [i['domain'] for i in get_domains()] == []