mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Feature/archive search (#70)
* Working through archive search. * Using custom hook. * Setting archive.contents duration creation. * Archive search and domain list use custom hooks. * Watching search param using value, not history. * Search params are preserved on load. Search and domain can be cleared. * Fixing archive external link. * Archive search form is new generic SearchInput. * Reworking Archives empty states. * Simplifying Archives logic. * Fixing bug which displayed the wrong empty state during loading. * Archives can be synced and deleted. * Changing wording of archive delete. * Removing URL table. * Deleting domain if it has no archives. * Fixing FE use of archive.url. Fixing bug with get_domains. * Universal results are displayed on Welcome page. * Creating ExternalLink to open a link in a new tab.g * Increasing size of search headers. * Adding divider to ease reading. * Adding More button below search results. * Disabling More buttons when there are no results. * Ordering videos by rank so linked results are at the top. Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
parent
b594f9761a
commit
c23b014add
17 changed files with 666 additions and 904 deletions
38
alembic/versions/53f6780aa599_drop_urls.py
Normal file
38
alembic/versions/53f6780aa599_drop_urls.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
"""Drop URLs.
|
||||
|
||||
Revision ID: 53f6780aa599
|
||||
Revises: 31896cc29b71
|
||||
Create Date: 2022-01-03 12:25:51.575505
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
from modules.archive.models import Archive
|
||||
|
||||
revision = '53f6780aa599'
|
||||
down_revision = '31896cc29b71'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
urls = {i['id']: i['url'] for i in session.execute('SELECT id, url FROM url')}
|
||||
archives = {i['id']: i['url_id'] for i in session.execute('SELECT id, url_id FROM archive')}
|
||||
|
||||
session.execute('ALTER TABLE archive ADD COLUMN url TEXT')
|
||||
|
||||
for archive_id, url_id in archives.items():
|
||||
archive = session.query(Archive).filter_by(id=archive_id).one()
|
||||
archive.url = urls[url_id]
|
||||
|
||||
session.execute('ALTER TABLE archive DROP COLUMN url_id')
|
||||
session.execute('DROP TABLE url')
|
||||
|
|
@ -39,7 +39,7 @@ pre.wrap-text {
|
|||
text-decoration: none !important;
|
||||
}
|
||||
|
||||
.video-card-link {
|
||||
.card-link {
|
||||
color: black;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,32 +1,83 @@
|
|||
import React from 'react';
|
||||
import React, {useEffect, useState} from 'react';
|
||||
import './App.css';
|
||||
import {NavBar} from "./components/Nav";
|
||||
import {Route, Switch} from "react-router-dom";
|
||||
import {FavoriteVideosPreview, VideosRoute, VideoWrapper, ViewedVideosPreview} from "./components/Videos";
|
||||
import {Route, Switch, useHistory} from "react-router-dom";
|
||||
import {VideosPreview, VideosRoute, VideoWrapper} from "./components/Videos";
|
||||
import Admin from "./components/Admin";
|
||||
import {Container, Divider, Header, Segment} from "semantic-ui-react";
|
||||
import {Container, Divider, Header} from "semantic-ui-react";
|
||||
import 'semantic-ui-offline/semantic.min.css';
|
||||
import {SemanticToastContainer} from 'react-semantic-toasts';
|
||||
import 'react-semantic-toasts/styles/react-semantic-alert.css';
|
||||
import {AppsRoute} from "./components/Apps";
|
||||
import {InventoryRoute} from "./components/Inventory";
|
||||
import {ArchiveRoute} from "./components/Archive";
|
||||
import {ArchiveRoute, ArchivesList} from "./components/Archive";
|
||||
import {Saver} from "./components/Upload";
|
||||
import {useSearchParam} from "./hooks/useSearchParam";
|
||||
import {searchArchives, searchVideos} from "./api";
|
||||
import {MoreButton, SearchInput} from "./components/Common";
|
||||
|
||||
const useSearch = () => {
|
||||
let [searchStr, setSearchStr] = useSearchParam('q');
|
||||
|
||||
const [archives, setArchives] = useState();
|
||||
const [videos, setVideos] = useState();
|
||||
|
||||
const localSearchArchives = async (term) => {
|
||||
setArchives(null);
|
||||
const [archives, total] = await searchArchives(0, 6, null, term);
|
||||
setArchives(archives);
|
||||
}
|
||||
|
||||
const localSearchVideos = async (term) => {
|
||||
setVideos(null);
|
||||
const [videos, total] = await searchVideos(0, 6, null, term);
|
||||
setVideos(videos);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
localSearchArchives(searchStr);
|
||||
localSearchVideos(searchStr)
|
||||
}, [searchStr]);
|
||||
|
||||
return {searchStr, setSearchStr, archives, videos}
|
||||
}
|
||||
|
||||
function Welcome() {
|
||||
return (
|
||||
<Container style={{marginTop: '2em'}}>
|
||||
const {searchStr, setSearchStr, archives, videos} = useSearch();
|
||||
const history = useHistory();
|
||||
|
||||
let body = (
|
||||
<>
|
||||
<Header as='h2'>Save your media</Header>
|
||||
<Saver/>
|
||||
</>
|
||||
);
|
||||
|
||||
if (searchStr) {
|
||||
let archiveMore = () => {
|
||||
history.push(`/archive?q=${searchStr}`);
|
||||
};
|
||||
let videosMore = () => {
|
||||
history.push(`/videos?q=${searchStr}&o=rank`);
|
||||
};
|
||||
|
||||
body = (<>
|
||||
<Header as='h2'>Archives</Header>
|
||||
<ArchivesList archives={archives} searchStr={searchStr}/>
|
||||
<MoreButton onClick={archiveMore} disabled={!archives || archives.length === 0}/>
|
||||
|
||||
<Divider/>
|
||||
|
||||
<Segment>
|
||||
<FavoriteVideosPreview/>
|
||||
</Segment>
|
||||
<Segment>
|
||||
<ViewedVideosPreview/>
|
||||
</Segment>
|
||||
<Header as='h2'>Videos</Header>
|
||||
<VideosPreview videos={videos}/>
|
||||
<MoreButton onClick={videosMore} disabled={!videos || videos.length === 0}/>
|
||||
</>);
|
||||
}
|
||||
|
||||
return (
|
||||
<Container style={{marginTop: '2em'}}>
|
||||
<SearchInput initValue={searchStr} onSubmit={setSearchStr} size='big'/>
|
||||
{body}
|
||||
</Container>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -307,16 +307,11 @@ export async function deleteItems(itemIds) {
|
|||
await apiDelete(`${API_URI}/inventory/item/${i}`);
|
||||
}
|
||||
|
||||
export async function postArchive(url) {
|
||||
let i = {url: url};
|
||||
return await apiPost(`${API_URI}/archive`, i);
|
||||
}
|
||||
|
||||
export async function deleteArchive(archive_id) {
|
||||
return await apiDelete(`${API_URI}/archive/${archive_id}`);
|
||||
}
|
||||
|
||||
export async function searchArchives(offset, limit, domain = null) {
|
||||
export async function searchArchives(offset, limit, domain = null, searchStr = null) {
|
||||
// Build a search query to retrieve a list of videos from the API
|
||||
offset = offset || 0;
|
||||
limit = limit || DEFAULT_LIMIT;
|
||||
|
|
@ -324,6 +319,9 @@ export async function searchArchives(offset, limit, domain = null) {
|
|||
if (domain) {
|
||||
body['domain'] = domain;
|
||||
}
|
||||
if (searchStr) {
|
||||
body['search_str'] = searchStr;
|
||||
}
|
||||
|
||||
let response = await apiPost(`${ARCHIVES_API}/search`, body);
|
||||
if (response.status === 200) {
|
||||
|
|
|
|||
|
|
@ -1,29 +1,31 @@
|
|||
import React from "react";
|
||||
import {Card, Confirm, Container, Form, Header, Icon, Image, Placeholder, Tab, Table} from "semantic-ui-react";
|
||||
import Paginator, {APIForm, objectToQuery, scrollToTop, uploadDate} from "./Common";
|
||||
import {deleteArchive, fetchDomains, postArchive, refreshArchives, searchArchives} from "../api";
|
||||
import React, {useState} from "react";
|
||||
import {Card, Confirm, Container, Icon, Image, Placeholder, Tab} from "semantic-ui-react";
|
||||
import Paginator, {ClearButton, ExternalLink, SearchInput, uploadDate} from "./Common";
|
||||
import {deleteArchive, postDownload, refreshArchives} from "../api";
|
||||
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
|
||||
import {Link, NavLink} from "react-router-dom";
|
||||
import * as QueryString from "query-string";
|
||||
import {ArchivePlaceholder} from "./Placeholder";
|
||||
import Table from "semantic-ui-react/dist/commonjs/collections/Table";
|
||||
import {useArchives} from "../hooks/useArchives";
|
||||
import {useDomains} from "../hooks/useDomains";
|
||||
|
||||
|
||||
function FailedArchiveCard({url, syncURL, deleteURL}) {
|
||||
function FailedArchiveCard({archive, syncArchive, deleteArchive}) {
|
||||
|
||||
let syncIcon = (
|
||||
<Button onClick={() => syncURL(url.url)}>
|
||||
<Button onClick={syncArchive}>
|
||||
<Icon name='sync' size='big'/>
|
||||
</Button>
|
||||
);
|
||||
|
||||
let trashIcon = (
|
||||
<Button onClick={() => deleteURL(url.id)}>
|
||||
<Button onClick={deleteArchive}>
|
||||
<Icon name='trash' size='big'/>
|
||||
</Button>
|
||||
);
|
||||
|
||||
let externalIcon = (
|
||||
<a href={url.url} target='_blank' rel='noopener noreferrer'>
|
||||
<a href={archive.url} target='_blank' rel='noopener noreferrer'>
|
||||
<Icon name='sign-out' size='big'/>
|
||||
</a>
|
||||
);
|
||||
|
|
@ -32,7 +34,7 @@ function FailedArchiveCard({url, syncURL, deleteURL}) {
|
|||
<Card>
|
||||
<Card.Content>
|
||||
<Card.Header>
|
||||
{url.url}
|
||||
{archive.url}
|
||||
</Card.Header>
|
||||
<Card.Description>
|
||||
<p>Failed!</p>
|
||||
|
|
@ -45,345 +47,172 @@ function FailedArchiveCard({url, syncURL, deleteURL}) {
|
|||
);
|
||||
}
|
||||
|
||||
class ArchiveCard extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
deleteOpen: false,
|
||||
syncOpen: false,
|
||||
}
|
||||
function ArchiveCard({archive, syncArchive, deleteArchive}) {
|
||||
const [syncOpen, setSyncOpen] = useState(false);
|
||||
const [deleteOpen, setDeleteOpen] = useState(false);
|
||||
|
||||
const localSyncArchive = async () => {
|
||||
setSyncOpen(false);
|
||||
await syncArchive(archive.url);
|
||||
}
|
||||
const localDeleteArchive = async () => {
|
||||
setDeleteOpen(false);
|
||||
await deleteArchive(archive.id);
|
||||
}
|
||||
|
||||
syncURL = async () => {
|
||||
await postArchive(this.props.archive.url.url);
|
||||
await this.props.fetchURLs();
|
||||
let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null;
|
||||
let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null;
|
||||
|
||||
if (archive.status === 'failed') {
|
||||
return <FailedArchiveCard archive={archive} syncArchive={localSyncArchive} deleteArchive={localDeleteArchive}/>;
|
||||
}
|
||||
|
||||
deleteURL = async () => {
|
||||
await deleteArchive(this.props.archive.id);
|
||||
await this.props.fetchURLs();
|
||||
let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null;
|
||||
let readabilityIcon = <Button icon disabled><Icon name='book' size='large'/></Button>;
|
||||
if (readabilityUrl) {
|
||||
readabilityIcon = (
|
||||
<Button icon href={readabilityUrl} target='_blank' rel='noopener noreferrer'>
|
||||
<Icon name='book' size='large'/>
|
||||
</Button>);
|
||||
}
|
||||
|
||||
render() {
|
||||
let archive = this.props.archive;
|
||||
let syncIcon = null;
|
||||
if (syncArchive) {
|
||||
syncIcon = (<>
|
||||
<Button icon onClick={() => setSyncOpen(true)}>
|
||||
<Icon name='sync' size='large'/>
|
||||
</Button>
|
||||
<Confirm
|
||||
open={syncOpen}
|
||||
content='Download the latest version of this URL?'
|
||||
confirmButton='Confirm'
|
||||
onCancel={() => setSyncOpen(false)}
|
||||
onConfirm={localSyncArchive}
|
||||
/>
|
||||
</>);
|
||||
}
|
||||
|
||||
let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null;
|
||||
let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null;
|
||||
|
||||
if (archive.status === 'failed') {
|
||||
return <FailedArchiveCard url={archive} syncURL={this.syncURL} deleteURL={this.deleteURL}/>;
|
||||
}
|
||||
|
||||
let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null;
|
||||
let readabilityIcon = <Button icon disabled><Icon name='book' size='large'/></Button>;
|
||||
if (readabilityUrl) {
|
||||
readabilityIcon = (
|
||||
<Button icon href={readabilityUrl} target='_blank' rel='noopener noreferrer'>
|
||||
<Icon name='book' size='large'/>
|
||||
</Button>);
|
||||
}
|
||||
|
||||
let syncIcon = (
|
||||
<>
|
||||
<Button icon onClick={() => this.setState({syncOpen: true})}>
|
||||
<Icon name='sync' size='large'/>
|
||||
let deleteIcon = null;
|
||||
if (deleteArchive) {
|
||||
deleteIcon = (<>
|
||||
<Button icon onClick={() => setDeleteOpen(true)}>
|
||||
<Icon name='trash' size='large'/>
|
||||
</Button>
|
||||
<Confirm
|
||||
open={this.state.syncOpen}
|
||||
content='Download the latest version of this URL?'
|
||||
confirmButton='Confirm'
|
||||
onCancel={() => this.setState({syncOpen: false})}
|
||||
onConfirm={this.syncURL}
|
||||
open={deleteOpen}
|
||||
content='Are you sure you want to delete this Archive? All files will be deleted.'
|
||||
confirmButton='Delete'
|
||||
onCancel={() => setDeleteOpen(false)}
|
||||
onConfirm={localDeleteArchive}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
|
||||
let deleteIcon = (
|
||||
<>
|
||||
<Button icon onClick={() => this.setState({deleteOpen: true})}>
|
||||
<Icon name='trash' size='large'/>
|
||||
</Button>
|
||||
<Confirm
|
||||
open={this.state.deleteOpen}
|
||||
content='Are you sure you want to delete this URL? All files will be deleted.'
|
||||
confirmButton='Delete'
|
||||
onCancel={() => this.setState({deleteOpen: false})}
|
||||
onConfirm={this.deleteURL}
|
||||
/>
|
||||
</>
|
||||
)
|
||||
;
|
||||
|
||||
let externalIcon = (
|
||||
<Button icon href={archive.url} target='_blank' rel='noopener noreferrer'>
|
||||
<Icon name='sign-out' size='large'/>
|
||||
</Button>
|
||||
);
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<Card.Content>
|
||||
<a href={singlefileUrl} target='_blank' rel='noopener noreferrer'>
|
||||
<Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>
|
||||
<Card.Header>
|
||||
<Container textAlign='left'>
|
||||
<p>{archive.title || archive.url}</p>
|
||||
</Container>
|
||||
</Card.Header>
|
||||
</a>
|
||||
<Card.Meta>
|
||||
{uploadDate(archive.archive_datetime)}
|
||||
</Card.Meta>
|
||||
<Card.Description>
|
||||
<Container textAlign='left'>
|
||||
{readabilityIcon}
|
||||
{syncIcon}
|
||||
{deleteIcon}
|
||||
{externalIcon}
|
||||
</Container>
|
||||
</Card.Description>
|
||||
</Card.Content>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
let externalIcon = (
|
||||
<Button icon href={archive.url} target='_blank' rel='noopener noreferrer'>
|
||||
<Icon name='sign-out' size='large'/>
|
||||
</Button>
|
||||
);
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
|
||||
<Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>
|
||||
</ExternalLink>
|
||||
<Card.Content>
|
||||
<Card.Header>
|
||||
<Container textAlign='left'>
|
||||
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
|
||||
<p>{archive.title || archive.url}</p>
|
||||
</ExternalLink>
|
||||
</Container>
|
||||
</Card.Header>
|
||||
<Card.Meta>
|
||||
{uploadDate(archive.archive_datetime)}
|
||||
</Card.Meta>
|
||||
<Card.Description>
|
||||
<Container textAlign='left'>
|
||||
{readabilityIcon}
|
||||
{syncIcon}
|
||||
{deleteIcon}
|
||||
{externalIcon}
|
||||
</Container>
|
||||
</Card.Description>
|
||||
</Card.Content>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
class ArchiveCards extends React.Component {
|
||||
render() {
|
||||
return (
|
||||
<Card.Group>
|
||||
{this.props.archives.map((i) => {
|
||||
return <ArchiveCard key={i['id']} archive={i} fetchURLs={this.props.fetchURLs}/>
|
||||
})}
|
||||
</Card.Group>
|
||||
)
|
||||
function ArchiveCards({archives, syncArchive, deleteArchive}) {
|
||||
return (
|
||||
<Card.Group>
|
||||
{archives.map((i) => {
|
||||
return <ArchiveCard key={i['id']} archive={i} syncArchive={syncArchive} deleteArchive={deleteArchive}/>
|
||||
})}
|
||||
</Card.Group>
|
||||
)
|
||||
}
|
||||
|
||||
export function ArchivesList({archives, searchStr, syncArchive, localDeleteArchive}) {
|
||||
if (archives === null || archives === undefined) {
|
||||
// Fetching archives.
|
||||
return <ArchivePlaceholder/>;
|
||||
} else if (archives.length === 0 && searchStr) {
|
||||
// Search with no results.
|
||||
return <p>No archives found! Is your search too restrictive?</p>;
|
||||
} else if (archives.length === 0) {
|
||||
// No archives fetched.
|
||||
return <p>No archives found! Have you archived any webpages?</p>;
|
||||
} else {
|
||||
// Archives fetched successfully!
|
||||
return <ArchiveCards archives={archives} syncArchive={syncArchive} deleteArchive={localDeleteArchive}/>;
|
||||
}
|
||||
}
|
||||
|
||||
class Archives extends React.Component {
|
||||
export function Archives() {
|
||||
const {archivesData, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} =
|
||||
useArchives();
|
||||
const {archives} = archivesData;
|
||||
|
||||
constructor(props) {
|
||||
super(props);
|
||||
const query = QueryString.parse(this.props.location.search);
|
||||
let activePage = query.page ? parseInt(query.page) : 1;
|
||||
let domain = query.domain || null;
|
||||
|
||||
this.state = {
|
||||
activePage: activePage,
|
||||
limit: 20,
|
||||
archives: null,
|
||||
totalPages: null,
|
||||
domain: domain,
|
||||
};
|
||||
this.fetchURLs = this.fetchURLs.bind(this);
|
||||
this.clearSearch = this.clearSearch.bind(this);
|
||||
this.changePage = this.changePage.bind(this);
|
||||
const syncArchive = async (url) => {
|
||||
await postDownload(url, 'archive');
|
||||
await search();
|
||||
}
|
||||
|
||||
async componentDidMount() {
|
||||
await this.fetchURLs();
|
||||
const localDeleteArchive = async (archive_id) => {
|
||||
await deleteArchive(archive_id);
|
||||
await search();
|
||||
}
|
||||
|
||||
async componentDidUpdate(prevProps, prevState, snapshot) {
|
||||
if (prevProps.location.search !== this.props.location.search) {
|
||||
await this.fetchURLs();
|
||||
}
|
||||
let domainClearButton = null;
|
||||
if (domain) {
|
||||
domainClearButton = <ClearButton
|
||||
onClick={() => setDomain(null)}
|
||||
style={{marginBottom: '1em'}}
|
||||
label={`Domain: ${domain}`}
|
||||
/>;
|
||||
}
|
||||
|
||||
async fetchURLs() {
|
||||
this.setState({archives: null});
|
||||
let offset = this.state.limit * this.state.activePage - this.state.limit;
|
||||
let [archives, total] = await searchArchives(offset, this.state.limit, this.state.domain);
|
||||
let totalPages = Math.round(total / this.state.limit) || 1;
|
||||
this.setState({archives: archives, totalPages: totalPages});
|
||||
}
|
||||
|
||||
clearSearch() {
|
||||
this.setState({activePage: 1, domain: null}, this.changePage);
|
||||
}
|
||||
|
||||
changePage(activePage) {
|
||||
if (activePage) {
|
||||
this.setState({activePage});
|
||||
}
|
||||
let {history, location} = this.props;
|
||||
|
||||
let search = {
|
||||
page: activePage > 1 ? activePage : null,
|
||||
domain: this.state.domain,
|
||||
};
|
||||
|
||||
history.push({
|
||||
pathname: location.pathname,
|
||||
search: objectToQuery(search),
|
||||
});
|
||||
scrollToTop();
|
||||
}
|
||||
|
||||
render() {
|
||||
let {archives, activePage, totalPages} = this.state;
|
||||
|
||||
if (archives === null) {
|
||||
return (<>
|
||||
<Header as='h1'>Latest Archives</Header>
|
||||
<ArchivePlaceholder/>
|
||||
</>)
|
||||
}
|
||||
|
||||
let pagination = null;
|
||||
if (totalPages) {
|
||||
pagination = (
|
||||
<div style={{marginTop: '3em', textAlign: 'center'}}>
|
||||
<Paginator
|
||||
activePage={activePage}
|
||||
changePage={this.changePage}
|
||||
totalPages={totalPages}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
let domainButton = null;
|
||||
if (this.state.domain) {
|
||||
domainButton = (
|
||||
<Button icon labelPosition='right' onClick={this.clearSearch} style={{marginBottom: '1em'}}>
|
||||
Search: {this.state.domain}
|
||||
<Icon name='close'/>
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Header as='h1'>Latest Archives</Header>
|
||||
{domainButton}
|
||||
<ArchiveCards archives={archives} fetchURLs={this.fetchURLs}/>
|
||||
{pagination}
|
||||
</>
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class Domains extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
domains: null,
|
||||
total: null,
|
||||
};
|
||||
this.fetchDomains = this.fetchDomains.bind(this);
|
||||
}
|
||||
|
||||
async componentDidMount() {
|
||||
await this.fetchDomains();
|
||||
}
|
||||
|
||||
async fetchDomains() {
|
||||
this.setState({domains: null});
|
||||
let [domains, total] = await fetchDomains();
|
||||
this.setState({domains, total});
|
||||
}
|
||||
|
||||
row(domain) {
|
||||
return <Table.Row key={domain['domain']}>
|
||||
<Table.Cell>
|
||||
<Link to={`/archive?domain=${domain['domain']}`}>
|
||||
{domain['domain']}
|
||||
</Link>
|
||||
</Table.Cell>
|
||||
<Table.Cell>{domain['url_count']}</Table.Cell>
|
||||
</Table.Row>
|
||||
}
|
||||
|
||||
render() {
|
||||
if (this.state.domains) {
|
||||
return (
|
||||
<>
|
||||
<Header as='h1'>Domains</Header>
|
||||
<Table celled>
|
||||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell>Domain</Table.HeaderCell>
|
||||
<Table.HeaderCell>URLs</Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
|
||||
<Table.Body>
|
||||
{this.state.domains.map(this.row)}
|
||||
</Table.Body>
|
||||
</Table>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
return (<>
|
||||
<Header as='h1'>Domains</Header>
|
||||
<Placeholder>
|
||||
<Placeholder.Header>
|
||||
<Placeholder.Line/>
|
||||
<Placeholder.Line/>
|
||||
</Placeholder.Header>
|
||||
</Placeholder>
|
||||
</>)
|
||||
}
|
||||
}
|
||||
|
||||
class ArchiveAddForm extends APIForm {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.archivesRef = props.archivesRef;
|
||||
|
||||
this.state = {
|
||||
...this.state,
|
||||
inputs: {
|
||||
url: '',
|
||||
},
|
||||
errors: {},
|
||||
};
|
||||
this.fetchURLs = this.fetchURLs.bind(this);
|
||||
}
|
||||
|
||||
handleSubmit = async (e) => {
|
||||
e.preventDefault();
|
||||
this.setLoading();
|
||||
try {
|
||||
let response = await postArchive(this.state.inputs.url);
|
||||
if (response.status === 204) {
|
||||
this.setState({inputs: {url: ''}, success: true}, this.fetchURLs);
|
||||
} else {
|
||||
this.setState({loading: false, success: undefined, error: true});
|
||||
}
|
||||
} finally {
|
||||
this.clearLoading();
|
||||
}
|
||||
}
|
||||
|
||||
fetchURLs = async (e) => {
|
||||
await this.archivesRef.current.fetchURLs();
|
||||
}
|
||||
|
||||
render() {
|
||||
return (
|
||||
<>
|
||||
<Form onSubmit={this.handleSubmit}>
|
||||
<label htmlFor='url'>Archive URL</label>
|
||||
<Form.Group>
|
||||
<Form.Input
|
||||
name='url'
|
||||
placeholder='https://wrolpi.org'
|
||||
value={this.state.inputs.url}
|
||||
disabled={this.state.disabled}
|
||||
error={this.state.error}
|
||||
onChange={this.handleInputChange}
|
||||
success={this.state.success}
|
||||
/>
|
||||
<Form.Button primary disabled={this.state.disabled}>Archive</Form.Button>
|
||||
</Form.Group>
|
||||
</Form>
|
||||
</>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<SearchInput initValue={searchStr} onSubmit={setSearchStr}/>
|
||||
{domainClearButton}
|
||||
<ArchivesList
|
||||
archives={archives}
|
||||
searchStr={searchStr}
|
||||
syncArchive={syncArchive}
|
||||
localDeleteArchive={localDeleteArchive}
|
||||
/>
|
||||
<div style={{marginTop: '3em', textAlign: 'center'}}>
|
||||
<Paginator
|
||||
activePage={activePage}
|
||||
changePage={setPage}
|
||||
totalPages={totalPages}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
class ManageArchives extends React.Component {
|
||||
|
|
@ -401,6 +230,51 @@ class ManageArchives extends React.Component {
|
|||
}
|
||||
}
|
||||
|
||||
function Domains() {
|
||||
|
||||
const [domains] = useDomains();
|
||||
|
||||
const row = (domain) => {
|
||||
return <Table.Row key={domain['domain']}>
|
||||
<Table.Cell>
|
||||
<Link to={`/archive?domain=${domain['domain']}`}>
|
||||
{domain['domain']}
|
||||
</Link>
|
||||
</Table.Cell>
|
||||
<Table.Cell>{domain['url_count']}</Table.Cell>
|
||||
</Table.Row>
|
||||
}
|
||||
|
||||
if (domains) {
|
||||
return (
|
||||
<>
|
||||
<Table celled>
|
||||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell>Domain</Table.HeaderCell>
|
||||
<Table.HeaderCell>URLs</Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
|
||||
<Table.Body>
|
||||
{domains.map(row)}
|
||||
</Table.Body>
|
||||
</Table>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
return (<>
|
||||
<Placeholder>
|
||||
<Placeholder.Header>
|
||||
<Placeholder.Line/>
|
||||
<Placeholder.Line/>
|
||||
</Placeholder.Header>
|
||||
</Placeholder>
|
||||
</>)
|
||||
}
|
||||
|
||||
|
||||
export class ArchiveRoute extends React.Component {
|
||||
|
||||
constructor(props) {
|
||||
|
|
@ -410,7 +284,43 @@ export class ArchiveRoute extends React.Component {
|
|||
this.state = {
|
||||
activeIndex: this.matchPaneTo(),
|
||||
}
|
||||
this.archivesRef = React.createRef();
|
||||
|
||||
this.archivePanes = [
|
||||
{
|
||||
menuItem: {
|
||||
as: NavLink,
|
||||
content: 'Archives',
|
||||
id: 'archive',
|
||||
to: '/archive',
|
||||
exact: true,
|
||||
key: 'home',
|
||||
},
|
||||
render: () => <Tab.Pane><Archives/></Tab.Pane>
|
||||
},
|
||||
{
|
||||
menuItem: {
|
||||
as: NavLink,
|
||||
content: 'Domains',
|
||||
id: 'domains',
|
||||
to: '/archive/domains',
|
||||
exact: true,
|
||||
key: 'domains',
|
||||
},
|
||||
render: () => <Tab.Pane><Domains/></Tab.Pane>
|
||||
},
|
||||
{
|
||||
menuItem: {
|
||||
as: NavLink,
|
||||
content: 'Manage',
|
||||
id: 'manage',
|
||||
to: '/archive/manage',
|
||||
exact: true,
|
||||
key: 'manage',
|
||||
},
|
||||
render: () => <Tab.Pane><ManageArchives/></Tab.Pane>
|
||||
},
|
||||
];
|
||||
|
||||
}
|
||||
|
||||
componentDidUpdate(prevProps, prevState, snapshot) {
|
||||
|
|
@ -424,46 +334,9 @@ export class ArchiveRoute extends React.Component {
|
|||
}
|
||||
|
||||
render() {
|
||||
const panes = [
|
||||
{
|
||||
menuItem: {
|
||||
as: NavLink,
|
||||
content: 'Archives',
|
||||
id: 'archive',
|
||||
to: '/archive',
|
||||
exact: true,
|
||||
key: 'home',
|
||||
},
|
||||
render: () => <Tab.Pane><Archives {...this.props} ref={this.archivesRef}/></Tab.Pane>
|
||||
},
|
||||
{
|
||||
menuItem: {
|
||||
as: NavLink,
|
||||
content: 'Domains',
|
||||
id: 'domains',
|
||||
to: '/archive/domains',
|
||||
exact: true,
|
||||
key: 'domains',
|
||||
},
|
||||
render: () => <Tab.Pane><Domains {...this.props}/></Tab.Pane>
|
||||
},
|
||||
{
|
||||
menuItem: {
|
||||
as: NavLink,
|
||||
content: 'Manage',
|
||||
id: 'manage',
|
||||
to: '/archive/manage',
|
||||
exact: true,
|
||||
key: 'manage',
|
||||
},
|
||||
render: () => <Tab.Pane><ManageArchives {...this.props}/></Tab.Pane>
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<Container style={{marginTop: '2em', marginBottom: '2em'}}>
|
||||
<ArchiveAddForm archivesRef={this.archivesRef}/>
|
||||
<Tab panes={panes} activeIndex={this.state.activeIndex}/>
|
||||
<Tab panes={this.archivePanes} activeIndex={this.state.activeIndex} renderActiveOnly={true}/>
|
||||
</Container>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import React from "react";
|
||||
import {Card, Container, Image, Pagination, Progress} from 'semantic-ui-react';
|
||||
import React, {useState} from "react";
|
||||
import {Card, Container, Form, Icon, Image, Input, Pagination, Progress} from 'semantic-ui-react';
|
||||
import {Link} from "react-router-dom";
|
||||
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
|
||||
export const API_URI = `http://${window.location.host}/api`;
|
||||
export const VIDEOS_API = `${API_URI}/videos`;
|
||||
|
|
@ -99,7 +101,7 @@ export function VideoCard({video}) {
|
|||
let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null;
|
||||
|
||||
return (
|
||||
<Card style={{'width': '18em', 'margin': '1em'}}>
|
||||
<Card>
|
||||
<Link to={video_url}>
|
||||
<Image src={poster_url} wrapped style={{position: 'relative', width: '100%'}}/>
|
||||
</Link>
|
||||
|
|
@ -107,7 +109,7 @@ export function VideoCard({video}) {
|
|||
<Card.Content>
|
||||
<Card.Header>
|
||||
<Container textAlign='left'>
|
||||
<Link to={video_url} className="no-link-underscore video-card-link">
|
||||
<Link to={video_url} className="no-link-underscore card-link">
|
||||
<p>{video.title || video.stem || video.video_path}</p>
|
||||
</Link>
|
||||
</Container>
|
||||
|
|
@ -115,7 +117,7 @@ export function VideoCard({video}) {
|
|||
<Card.Description>
|
||||
<Container textAlign='left'>
|
||||
{channel &&
|
||||
<Link to={channel_url} className="no-link-underscore video-card-link">
|
||||
<Link to={channel_url} className="no-link-underscore card-link">
|
||||
<b>{channel.name}</b>
|
||||
</Link>}
|
||||
<p>{upload_date}</p>
|
||||
|
|
@ -126,17 +128,14 @@ export function VideoCard({video}) {
|
|||
)
|
||||
}
|
||||
|
||||
export class VideoCards extends React.Component {
|
||||
|
||||
render() {
|
||||
return (
|
||||
<Card.Group>
|
||||
{this.props.videos.map((v) => {
|
||||
return <VideoCard key={v['id']} video={v}/>
|
||||
})}
|
||||
</Card.Group>
|
||||
)
|
||||
}
|
||||
export function VideoCards({videos}) {
|
||||
return (
|
||||
<Card.Group>
|
||||
{videos.map((v) => {
|
||||
return <VideoCard key={v['id']} video={v}/>
|
||||
})}
|
||||
</Card.Group>
|
||||
)
|
||||
}
|
||||
|
||||
export function RequiredAsterisk() {
|
||||
|
|
@ -490,3 +489,88 @@ export function objectToQuery(d) {
|
|||
arr = arr.filter((i) => i !== '')
|
||||
return arr.join('&');
|
||||
}
|
||||
|
||||
export function ClearButton({onClick, style, label, icon = 'close'}) {
|
||||
return <Button icon
|
||||
labelPosition='right'
|
||||
onClick={onClick}
|
||||
style={style}
|
||||
>
|
||||
{label}
|
||||
<Icon name={icon}/>
|
||||
</Button>
|
||||
}
|
||||
|
||||
export function SearchInput({initValue, onSubmit, size}) {
|
||||
let [value, setValue] = useState(initValue || '');
|
||||
size = size || 'small';
|
||||
|
||||
const buttonClick = () => {
|
||||
// Clear the input when the "clear" button is clicked, search again.
|
||||
setValue('');
|
||||
onSubmit(value);
|
||||
}
|
||||
|
||||
// Button is "search" when input is dirty, otherwise it is "clear".
|
||||
let button = (
|
||||
<Button icon onClick={buttonClick} style={{marginLeft: '0.5em'}} type='reset' size={size}>
|
||||
<Icon name='close'/>
|
||||
</Button>
|
||||
);
|
||||
if (!initValue || initValue !== value) {
|
||||
button = (
|
||||
<Button icon style={{marginLeft: '0.5em'}} type='submit' disabled={value === ''} size={size}>
|
||||
<Icon name='search'/>
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
||||
const localOnSubmit = (e) => {
|
||||
// Send the value up when submitting.
|
||||
e.preventDefault();
|
||||
onSubmit(value);
|
||||
}
|
||||
|
||||
return (
|
||||
<Form onSubmit={localOnSubmit}>
|
||||
<Form.Group inline style={{marginBottom: '1em'}}>
|
||||
<Input
|
||||
type='text'
|
||||
placeholder='Search...'
|
||||
onChange={(e) => setValue(e.target.value)}
|
||||
value={value}
|
||||
size={size}
|
||||
/>
|
||||
{button}
|
||||
</Form.Group>
|
||||
</Form>
|
||||
)
|
||||
}
|
||||
|
||||
export function ExternalLink(props) {
|
||||
return (
|
||||
<Link to={props.to} target='_blank' className={props.className} rel='noopener noreferrer'>
|
||||
{props.children}
|
||||
</Link>
|
||||
)
|
||||
}
|
||||
|
||||
export function MoreButton(props) {
|
||||
return (
|
||||
<Grid columns={1} textAlign='center'>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<Button
|
||||
onClick={props.onClick}
|
||||
size='big'
|
||||
style={{marginTop: '1em'}}
|
||||
disabled={props.disabled}
|
||||
>
|
||||
{props.children}
|
||||
More
|
||||
</Button>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,62 +137,16 @@ export class VideoWrapper extends React.Component {
|
|||
}
|
||||
}
|
||||
|
||||
class VideosPreview extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
videos: null,
|
||||
};
|
||||
export function VideosPreview({videos}) {
|
||||
let body = <VideoPlaceholder/>
|
||||
if (videos && videos.length === 0) {
|
||||
body = <p>No videos available.</p>
|
||||
} else if (videos && videos.length > 0) {
|
||||
body = <VideoCards videos={videos}/>
|
||||
}
|
||||
|
||||
async componentDidMount() {
|
||||
await this.fetchVideos();
|
||||
}
|
||||
|
||||
render() {
|
||||
let header = <h2>{this.title}</h2>;
|
||||
let body = <VideoPlaceholder/>
|
||||
if (this.state.videos && this.state.videos.length === 0) {
|
||||
body = <p>No videos available.</p>
|
||||
} else if (this.state.videos && this.state.videos.length > 0) {
|
||||
body = <VideoCards videos={this.state.videos}/>
|
||||
}
|
||||
return <>
|
||||
{header}
|
||||
{body}
|
||||
</>
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class FavoriteVideosPreview extends VideosPreview {
|
||||
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.title = 'Favorite Videos'
|
||||
}
|
||||
|
||||
async fetchVideos() {
|
||||
let [videos] = await searchVideos(
|
||||
0, 3, null, null, '-favorite', ['favorite']);
|
||||
this.setState({videos});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class ViewedVideosPreview extends VideosPreview {
|
||||
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.title = 'Recently Viewed Videos'
|
||||
}
|
||||
|
||||
async fetchVideos() {
|
||||
let [videos] = await searchVideos(
|
||||
0, 3, null, null, '-viewed');
|
||||
this.setState({videos});
|
||||
}
|
||||
|
||||
return <>
|
||||
{body}
|
||||
</>
|
||||
}
|
||||
|
||||
function FilterModal(props) {
|
||||
|
|
|
|||
52
app/src/hooks/useArchives.js
Normal file
52
app/src/hooks/useArchives.js
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import {useEffect, useState} from "react";
|
||||
import {searchArchives} from "../api";
|
||||
import {useSearchParam} from "./useSearchParam";
|
||||
|
||||
|
||||
export const useArchives = (defaultLimit = 20) => {
|
||||
const [archivesData, setArchives] = useState({archives: null, total: 0});
|
||||
const [totalPages, setTotalPages] = useState(0);
|
||||
const [activePage, setActivePage] = useState(1);
|
||||
|
||||
let [offset, setOffset] = useSearchParam('o');
|
||||
let [limit, setLimit] = useSearchParam('l', defaultLimit);
|
||||
let [domain, setDomain] = useSearchParam('domain');
|
||||
let [searchStr, setSearchStr] = useSearchParam('q');
|
||||
|
||||
const setPage = (i) => {
|
||||
i = parseInt(i);
|
||||
let l = parseInt(limit);
|
||||
setOffset((l * i) - l);
|
||||
setActivePage(i);
|
||||
}
|
||||
|
||||
const search = async (term) => {
|
||||
setArchives({archives: null, total: 0});
|
||||
setTotalPages(0);
|
||||
const [archives, total] = await searchArchives(offset, limit, domain, term);
|
||||
setTotalPages(Math.floor(total / limit) + 1);
|
||||
setArchives({archives, total});
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
search(searchStr);
|
||||
}, [offset, limit, domain, searchStr]);
|
||||
|
||||
return {
|
||||
archivesData,
|
||||
totalPages,
|
||||
setTotalPages,
|
||||
offset,
|
||||
setOffset,
|
||||
setPage,
|
||||
limit,
|
||||
setLimit,
|
||||
domain,
|
||||
setDomain,
|
||||
searchStr,
|
||||
setSearchStr,
|
||||
activePage,
|
||||
setActivePage,
|
||||
search,
|
||||
};
|
||||
}
|
||||
21
app/src/hooks/useDomains.js
Normal file
21
app/src/hooks/useDomains.js
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import {useEffect, useState} from "react";
|
||||
import {fetchDomains} from "../api";
|
||||
|
||||
export const useDomains = () => {
|
||||
const [domains, setDomains] = useState(null);
|
||||
const [total, setTotal] = useState(null);
|
||||
|
||||
const _fetchDomains = async () => {
|
||||
setDomains(null);
|
||||
setTotal(0);
|
||||
let [domains, total] = await fetchDomains();
|
||||
setDomains(domains);
|
||||
setTotal(total);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
_fetchDomains();
|
||||
}, []);
|
||||
|
||||
return [domains, total];
|
||||
}
|
||||
21
app/src/hooks/useSearchParam.js
Normal file
21
app/src/hooks/useSearchParam.js
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import {useEffect, useState} from "react";
|
||||
import {useHistory} from "react-router-dom";
|
||||
|
||||
export const useSearchParam = (key, defaultValue = null) => {
|
||||
// Get a window.location.search param.
|
||||
const startingValue = new URLSearchParams(window.location.search).get(key);
|
||||
|
||||
const [value, setValue] = useState(startingValue || defaultValue);
|
||||
const history = useHistory();
|
||||
|
||||
useEffect(() => {
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
params.delete(key);
|
||||
if (value) {
|
||||
params.append(key, value);
|
||||
}
|
||||
history.push({search: params.toString()});
|
||||
}, [value])
|
||||
|
||||
return [value, setValue];
|
||||
}
|
||||
|
|
@ -30,7 +30,7 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
if download.attempts > 3:
|
||||
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
|
||||
|
||||
lib.new_archive(download.url, sync=True)
|
||||
lib.do_archive(download.url)
|
||||
return True
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,7 @@ from wrolpi.errors import ValidationError
|
|||
from wrolpi.root_api import get_blueprint, json_response
|
||||
from wrolpi.schema import validate_doc, JSONErrorResponse
|
||||
from . import lib
|
||||
from .schema import RetrieveURLsRequest, RetrieveURLsResponse, PostArchiveRequest, ArchiveSearchRequest, \
|
||||
ArchiveSearchResponse
|
||||
from .schema import ArchiveSearchRequest, ArchiveSearchResponse
|
||||
|
||||
NAME = 'archive'
|
||||
|
||||
|
|
@ -19,22 +18,9 @@ bp = get_blueprint('Archive', '/api/archive')
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
@bp.post('/')
|
||||
@validate_doc(
|
||||
'Archive a website',
|
||||
PostArchiveRequest,
|
||||
)
|
||||
@wrol_mode_check
|
||||
async def post_archive(_: Request, data: dict):
|
||||
# Remove whitespace from URL.
|
||||
url = data['url'].strip()
|
||||
lib.new_archive(url)
|
||||
return response.empty()
|
||||
|
||||
|
||||
@bp.delete('/<archive_id:int>')
|
||||
@validate_doc(
|
||||
'Delete a website record',
|
||||
'Delete an individual archive',
|
||||
responses=[
|
||||
(HTTPStatus.NOT_FOUND, JSONErrorResponse),
|
||||
]
|
||||
|
|
@ -45,27 +31,6 @@ async def delete_archive(_: Request, archive_id: int):
|
|||
return response.empty()
|
||||
|
||||
|
||||
@bp.post('/urls')
|
||||
@validate_doc(
|
||||
'Get a list of URLs',
|
||||
RetrieveURLsRequest,
|
||||
RetrieveURLsResponse,
|
||||
)
|
||||
async def get_urls(_: Request, data: dict):
|
||||
try:
|
||||
limit = abs(int(data.get('limit', 20)))
|
||||
offset = abs(int(data.get('offset', 0)))
|
||||
domain = data.get('domain')
|
||||
except Exception as e:
|
||||
logger.error(f'Bad request', exc_info=e)
|
||||
return response.json(dict(error='bad request'), HTTPStatus.BAD_REQUEST)
|
||||
|
||||
urls = lib.get_urls(limit, offset, domain)
|
||||
count = lib.get_url_count(domain)
|
||||
ret = dict(urls=urls, totals=dict(urls=count))
|
||||
return json_response(ret)
|
||||
|
||||
|
||||
@bp.post('/refresh')
|
||||
@wrol_mode_check
|
||||
async def refresh_archives(_: Request):
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from uuid import uuid4
|
|||
|
||||
import pytest
|
||||
|
||||
from modules.archive.models import Archive, URL, Domain
|
||||
from modules.archive.models import Archive, Domain
|
||||
from wrolpi.dates import strftime_ms
|
||||
|
||||
|
||||
|
|
@ -58,32 +58,20 @@ def archive_factory(test_session, archive_directory):
|
|||
test_session.add(domain)
|
||||
test_session.flush([domain])
|
||||
|
||||
url_ = test_session.query(URL).filter_by(url=url).one_or_none()
|
||||
if not url_:
|
||||
url_ = URL(
|
||||
url=url,
|
||||
domain_id=domain.id,
|
||||
)
|
||||
test_session.add(url_)
|
||||
test_session.flush([url_])
|
||||
|
||||
archive = Archive(
|
||||
readability_json_path=readability_json_path,
|
||||
readability_path=readability_path,
|
||||
readability_txt_path=readability_txt_path,
|
||||
screenshot_path=screenshot_path,
|
||||
singlefile_path=singlefile_path,
|
||||
status='complete',
|
||||
title=title,
|
||||
url_id=url_.id,
|
||||
url=url,
|
||||
domain_id=domain.id,
|
||||
contents=contents,
|
||||
archive_datetime=timestamp,
|
||||
)
|
||||
test_session.add(archive)
|
||||
test_session.flush([archive])
|
||||
|
||||
url_.latest_id = archive.id
|
||||
test_session.commit()
|
||||
|
||||
for path in archive.my_paths():
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import gzip
|
||||
import json
|
||||
|
|
@ -11,12 +10,11 @@ import requests
|
|||
from bs4 import BeautifulSoup
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.archive.models import URL, Domain, Archive
|
||||
from modules.archive.models import Domain, Archive
|
||||
from wrolpi.common import get_media_directory, logger, chunks, extract_domain
|
||||
from wrolpi.dates import now, strptime_ms, strftime_ms
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
|
||||
from wrolpi.errors import InvalidDomain, UnknownURL, PendingArchive, InvalidArchive
|
||||
from wrolpi.vars import PYTEST
|
||||
from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -109,122 +107,71 @@ def request_archive(url: str):
|
|||
return singlefile, readability, screenshot
|
||||
|
||||
|
||||
def is_pending_archive(url: str) -> bool:
|
||||
with get_db_session() as session:
|
||||
url = session.query(URL).filter_by(url=url).one_or_none()
|
||||
if url and url.latest.status == 'pending':
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def new_archive(url: str, sync: bool = False):
|
||||
def do_archive(url: str) -> Archive:
|
||||
"""
|
||||
Request archiving of the provided URL. Store the returned files in their domain's directory.
|
||||
|
||||
:param url: The URL to archive.
|
||||
:param sync: Perform the archiving process synchronously for testing.
|
||||
Perform the real archive request to the archiving service. Store the resulting data into files. Create an Archive
|
||||
record in the DB. Create Domain/URL if missing.
|
||||
"""
|
||||
# Check that the archive files are available.
|
||||
get_new_archive_files(url)
|
||||
singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
|
||||
get_new_archive_files(url)
|
||||
|
||||
if is_pending_archive(url):
|
||||
raise PendingArchive()
|
||||
singlefile, readability, screenshot = request_archive(url)
|
||||
|
||||
# Store the single-file HTML in its own file.
|
||||
with singlefile_path.open('wt') as fh:
|
||||
fh.write(singlefile)
|
||||
if screenshot:
|
||||
with screenshot_path.open('wb') as fh:
|
||||
fh.write(screenshot)
|
||||
else:
|
||||
screenshot_path = None
|
||||
|
||||
# Store the Readability into separate files. This allows the user to view text-only or html articles.
|
||||
title = None
|
||||
readability_txt = None
|
||||
if readability:
|
||||
title = readability.get('title')
|
||||
|
||||
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
|
||||
with readability_path.open('wt') as fh:
|
||||
fh.write(readability.pop('content'))
|
||||
with readability_txt_path.open('wt') as fh:
|
||||
readability_txt = readability.pop('textContent')
|
||||
fh.write(readability_txt)
|
||||
else:
|
||||
readability_path = readability_txt_path = None
|
||||
|
||||
# Always write a JSON file that contains at least the URL.
|
||||
readability = readability or {}
|
||||
# Use the Readability title, or try and extract one from singlefile.
|
||||
if not title and singlefile:
|
||||
title = get_title_from_html(singlefile, url=url)
|
||||
readability['title'] = title
|
||||
readability['url'] = url
|
||||
with readability_json_path.open('wt') as fh:
|
||||
fh.write(json.dumps(readability))
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
domain, url_ = get_or_create_domain_and_url(session, url)
|
||||
|
||||
domain = get_or_create_domain(session, url)
|
||||
archive = Archive(
|
||||
url_id=url_.id,
|
||||
domain_id=domain.id,
|
||||
status='pending',
|
||||
title=title,
|
||||
archive_datetime=now(),
|
||||
singlefile_path=singlefile_path,
|
||||
readability_path=readability_path,
|
||||
readability_json_path=readability_json_path,
|
||||
readability_txt_path=readability_txt_path,
|
||||
screenshot_path=screenshot_path,
|
||||
contents=readability_txt,
|
||||
url=url,
|
||||
domain_id=domain.id,
|
||||
)
|
||||
session.add(archive)
|
||||
session.flush()
|
||||
archive_id = archive.id
|
||||
|
||||
url_.latest_id = archive_id
|
||||
url_.latest_datetime = now()
|
||||
|
||||
if PYTEST or sync:
|
||||
return _do_archive(url, archive_id)
|
||||
else:
|
||||
# Run the real archive process in the future.
|
||||
asyncio.ensure_future(do_archive(url, archive_id))
|
||||
|
||||
return archive
|
||||
|
||||
|
||||
def _do_archive(url: str, archive_id: int):
|
||||
"""
|
||||
Perform the real archive request to the archiving service. Store the resulting data into files. Update the Archive
|
||||
in the DB.
|
||||
"""
|
||||
try:
|
||||
singlefile, readability, screenshot = request_archive(url)
|
||||
|
||||
singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
|
||||
get_new_archive_files(url)
|
||||
|
||||
# Store the single-file HTML in it's own file.
|
||||
with singlefile_path.open('wt') as fh:
|
||||
fh.write(singlefile)
|
||||
if screenshot:
|
||||
with screenshot_path.open('wb') as fh:
|
||||
fh.write(screenshot)
|
||||
else:
|
||||
screenshot_path = None
|
||||
|
||||
# Store the Readability into separate files. This allows the user to view text-only or html articles.
|
||||
title = None
|
||||
if readability:
|
||||
title = readability.get('title')
|
||||
|
||||
# Write the readability parts to their own files. Write what is left after pops to the JSON file.
|
||||
with readability_path.open('wt') as fh:
|
||||
fh.write(readability.pop('content'))
|
||||
with readability_txt_path.open('wt') as fh:
|
||||
fh.write(readability.pop('textContent'))
|
||||
else:
|
||||
readability_path = readability_txt_path = None
|
||||
|
||||
# Always write a JSON file that contains at least the URL.
|
||||
readability = readability or {}
|
||||
# Use the Readability title, or try and extract one from singlefile.
|
||||
if not title and singlefile:
|
||||
title = get_title_from_html(singlefile, url=url)
|
||||
readability['title'] = title
|
||||
readability['url'] = url
|
||||
with readability_json_path.open('wt') as fh:
|
||||
fh.write(json.dumps(readability))
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
archive = session.query(Archive).filter_by(id=archive_id).one()
|
||||
archive.status = 'complete'
|
||||
archive.archive_datetime = now()
|
||||
archive.title = title
|
||||
archive.singlefile_path = singlefile_path
|
||||
archive.readability_path = readability_path
|
||||
archive.readability_json_path = readability_json_path
|
||||
archive.readability_txt_path = readability_txt_path
|
||||
archive.screenshot_path = screenshot_path
|
||||
# Update the latest for easy viewing.
|
||||
archive.url.latest_id = archive.id
|
||||
archive.url.latest_datetime = archive.archive_datetime
|
||||
|
||||
return archive
|
||||
except Exception:
|
||||
with get_db_session(commit=True) as session:
|
||||
archive = session.query(Archive).filter_by(id=archive_id).one()
|
||||
archive.status = 'failed'
|
||||
raise
|
||||
|
||||
|
||||
async def do_archive(url: str, archive_id: int):
|
||||
_do_archive(url, archive_id)
|
||||
|
||||
|
||||
def get_or_create_domain_and_url(session, url):
|
||||
def get_or_create_domain(session, url) -> Domain:
|
||||
"""
|
||||
Get/create the Domain for this archive.
|
||||
"""
|
||||
|
|
@ -234,12 +181,7 @@ def get_or_create_domain_and_url(session, url):
|
|||
domain = Domain(domain=domain_, directory=str(get_domain_directory(url)))
|
||||
session.add(domain)
|
||||
session.flush()
|
||||
url_ = session.query(URL).filter_by(url=url).one_or_none()
|
||||
if not url_:
|
||||
url_ = URL(url=url, domain_id=domain.id)
|
||||
session.add(url_)
|
||||
session.flush()
|
||||
return domain, url_
|
||||
return domain
|
||||
|
||||
|
||||
def get_title_from_html(html: str, url: str = None) -> str:
|
||||
|
|
@ -260,42 +202,6 @@ def get_domain(session, domain: str) -> Domain:
|
|||
return domain_
|
||||
|
||||
|
||||
def get_urls(limit: int = 20, offset: int = 0, domain: str = ''):
|
||||
with get_db_session() as session:
|
||||
if domain:
|
||||
domain_ = get_domain(session, domain)
|
||||
urls = domain_.urls[offset:offset + limit]
|
||||
urls = sorted(urls, key=lambda i: i.latest_datetime)[::-1]
|
||||
else:
|
||||
urls = session.query(URL) \
|
||||
.order_by(URL.latest_datetime.desc()) \
|
||||
.limit(limit) \
|
||||
.offset(offset) \
|
||||
.all()
|
||||
urls = [i.dict() for i in urls]
|
||||
return urls
|
||||
|
||||
|
||||
def get_url_count(domain: str = '') -> int:
|
||||
"""
|
||||
Get count of all URLs. Or, get count of all attached to a specific domain string.
|
||||
"""
|
||||
with get_db_session() as session:
|
||||
domain_id = None
|
||||
if domain:
|
||||
domain_id = get_domain(session, domain).id
|
||||
|
||||
with get_db_curs() as curs:
|
||||
stmt = 'SELECT COUNT(*) FROM url'
|
||||
params = {}
|
||||
if domain_id:
|
||||
stmt = f'{stmt} WHERE domain_id = %(domain_id)s'
|
||||
params['domain_id'] = domain_id
|
||||
curs.execute(stmt, params)
|
||||
count = int(curs.fetchone()[0])
|
||||
return count
|
||||
|
||||
|
||||
def delete_archive(archive_id: int):
|
||||
"""
|
||||
Delete an Archive and all of it's files.
|
||||
|
|
@ -307,14 +213,6 @@ def delete_archive(archive_id: int):
|
|||
|
||||
# Delete any files associated with this URL.
|
||||
archive.delete()
|
||||
# URL must have the latest
|
||||
url = archive.url
|
||||
url.update_latest()
|
||||
# Delete this archive.
|
||||
session.query(Archive).filter_by(id=archive_id).delete()
|
||||
# Remove the URL if necessary.
|
||||
if not url.archives:
|
||||
session.query(URL).filter_by(id=url.id).delete()
|
||||
|
||||
|
||||
def group_archive_files(files: Iterator[pathlib.Path]) -> groupby:
|
||||
|
|
@ -392,35 +290,19 @@ def _refresh_archives():
|
|||
if archive_count:
|
||||
logger.info(f'Inserted/updated {archive_count} archives')
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Check that each URLs most recent Archive exists, if it does not use the next most recent.
|
||||
urls = session.query(URL).all()
|
||||
for url in urls:
|
||||
if not url.latest or not url.latest.singlefile_path or not url.latest.singlefile_path.path.exists():
|
||||
url.update_latest()
|
||||
session.commit()
|
||||
|
||||
singlefile_paths = list(singlefile_paths)
|
||||
with get_db_curs(commit=True) as curs:
|
||||
curs.execute('DELETE FROM archive WHERE singlefile_path != ALL(%s)', (singlefile_paths,))
|
||||
|
||||
with get_db_curs(commit=True) as curs:
|
||||
stmt = '''
|
||||
DELETE FROM url WHERE id NOT IN (
|
||||
select distinct url_id from archive
|
||||
) RETURNING url.id
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
urls = list(map(dict, curs.fetchall()))
|
||||
stmt = '''
|
||||
DELETE FROM domains WHERE
|
||||
id NOT IN (select distinct domain_id from url)
|
||||
AND id NOT IN (select distinct domain_id from archive)
|
||||
id NOT IN (select distinct domain_id from archive)
|
||||
RETURNING domains.id
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
domains = list(map(dict, curs.fetchall()))
|
||||
logger.info(f'Deleted {len(urls)} URLS and {len(domains)} Domains')
|
||||
logger.info(f'Deleted {len(domains)} Domains')
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Get all archives that have no text contents, but have txt files.
|
||||
|
|
@ -450,40 +332,33 @@ def upsert_archive(dt: str, files, session: Session):
|
|||
except Exception as e:
|
||||
raise InvalidArchive() from e
|
||||
|
||||
domain, url_ = get_or_create_domain_and_url(session, url)
|
||||
logger.debug(f'Upsert url={url_.url}')
|
||||
domain = get_or_create_domain(session, url)
|
||||
# Get the existing Archive, or create a new one.
|
||||
archive = session.query(Archive).filter_by(singlefile_path=singlefile_path).one_or_none()
|
||||
if not archive:
|
||||
# No archive matches this singlefile_path, create a new one.
|
||||
archive = Archive(
|
||||
url_id=url_.id,
|
||||
url=url,
|
||||
domain_id=domain.id,
|
||||
)
|
||||
session.add(archive)
|
||||
session.flush()
|
||||
|
||||
# Update the archive with the files that we have.
|
||||
archive.status = 'complete',
|
||||
archive.archive_datetime = dt
|
||||
archive.singlefile_path = singlefile_path
|
||||
archive.readability_path = readability_path
|
||||
archive.readability_txt_path = readability_txt_path
|
||||
archive.readability_json_path = readability_json_path
|
||||
archive.screenshot_path = screenshot_path
|
||||
archive_id = archive.id
|
||||
|
||||
if not url_.latest_datetime or url_.latest_datetime < dt:
|
||||
url_.latest_id = archive_id
|
||||
url_.latest_datetime = archive.archive_datetime
|
||||
|
||||
|
||||
def get_domains():
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT domains.domain AS domain, COUNT(u.id) AS url_count
|
||||
SELECT domains.domain AS domain, COUNT(a.id) AS url_count
|
||||
FROM domains
|
||||
LEFT JOIN url u on domains.id = u.domain_id
|
||||
LEFT JOIN archive a on domains.id = a.domain_id
|
||||
GROUP BY domains.domain
|
||||
ORDER BY domains.domain
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ class Archive(Base, ModelHelper):
|
|||
__tablename__ = 'archive'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
url_id = Column(Integer, ForeignKey('url.id', ondelete='cascade'))
|
||||
url = relationship('URL', primaryjoin='Archive.url_id==URL.id')
|
||||
domain_id = Column(Integer, ForeignKey('domains.id'))
|
||||
domain = relationship('Domain', primaryjoin='Archive.domain_id==Domain.id')
|
||||
|
||||
|
|
@ -24,16 +22,16 @@ class Archive(Base, ModelHelper):
|
|||
readability_txt_path = Column(MediaPathType)
|
||||
screenshot_path = Column(MediaPathType)
|
||||
|
||||
url = Column(String)
|
||||
title = Column(String)
|
||||
archive_datetime = Column(TZDateTime)
|
||||
status = Column(String)
|
||||
contents = Column(String)
|
||||
|
||||
textsearch = Column(tsvector, Computed('''setweight(to_tsvector('english'::regconfig, title), 'A') ||
|
||||
setweight(to_tsvector('english'::regconfig, contents), 'D')'''))
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Archive id={self.id} url_id={self.url_id} singlefile={self.singlefile_path}>'
|
||||
return f'<Archive id={self.id} url={self.url} singlefile={self.singlefile_path}>'
|
||||
|
||||
def my_paths(self) -> Generator:
|
||||
if self.singlefile_path:
|
||||
|
|
@ -65,12 +63,6 @@ class Archive(Base, ModelHelper):
|
|||
return self.archive_datetime > other.archive_datetime
|
||||
|
||||
def __json__(self):
|
||||
url = None
|
||||
if self.url:
|
||||
url = dict(
|
||||
url=self.url.url,
|
||||
id=self.url.id,
|
||||
)
|
||||
d = dict(
|
||||
archive_datetime=self.archive_datetime,
|
||||
domain=self.domain.dict() if self.domain else None,
|
||||
|
|
@ -81,10 +73,8 @@ class Archive(Base, ModelHelper):
|
|||
readability_txt_path=self.readability_txt_path,
|
||||
screenshot_path=self.screenshot_path,
|
||||
singlefile_path=self.singlefile_path,
|
||||
status=self.status,
|
||||
title=self.title,
|
||||
url=url,
|
||||
url_id=self.url_id,
|
||||
url=self.url,
|
||||
)
|
||||
return d
|
||||
|
||||
|
|
@ -93,63 +83,12 @@ class Archive(Base, ModelHelper):
|
|||
|
||||
session = Session.object_session(self)
|
||||
|
||||
if self.url:
|
||||
self.url.update_latest()
|
||||
if not self.url.latest_id:
|
||||
self.url.delete()
|
||||
|
||||
session.query(Archive).filter_by(id=self.id).delete()
|
||||
|
||||
|
||||
class URL(Base, ModelHelper):
|
||||
__tablename__ = 'url'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
url = Column(String, unique=True)
|
||||
|
||||
archives: InstrumentedList = relationship('Archive', primaryjoin='URL.id==Archive.url_id')
|
||||
latest_id = Column(Integer, ForeignKey('archive.id'))
|
||||
latest = relationship('Archive', primaryjoin='URL.latest_id==Archive.id')
|
||||
latest_datetime = Column(TZDateTime)
|
||||
domain_id = Column(Integer, ForeignKey('domains.id'))
|
||||
domain = relationship('Domain', primaryjoin='URL.domain_id==Domain.id')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<URL id={self.id} url={self.url}>'
|
||||
|
||||
def dict(self) -> dict:
|
||||
d = super().dict()
|
||||
d['latest'] = self.latest.dict() if self.latest else None
|
||||
d['domain'] = self.domain.dict()
|
||||
d['archives'] = [i.dict() for i in self.archives]
|
||||
return d
|
||||
|
||||
def update_latest(self):
|
||||
"""
|
||||
Set `latest` to the most recent Archive of this URL whose `singlefile` exists.
|
||||
"""
|
||||
self.latest_id = None
|
||||
archives = filter(lambda i: i.singlefile_path and i.singlefile_path.path.exists(), self.archives)
|
||||
try:
|
||||
latest: Archive = max(archives)
|
||||
if latest:
|
||||
self.latest_id = latest.id
|
||||
except ValueError:
|
||||
# No archives exist!
|
||||
pass
|
||||
|
||||
def delete(self):
|
||||
session = Session.object_session(self)
|
||||
archive_ids = [int(i.id) for i in self.archives]
|
||||
if archive_ids:
|
||||
for id_ in archive_ids:
|
||||
session.query(Archive).filter_by(id=id_).delete()
|
||||
|
||||
domain = self.domain
|
||||
session.query(URL).filter_by(id=self.id).delete()
|
||||
|
||||
if len(domain.urls) == 0:
|
||||
domain.delete()
|
||||
if self.domain:
|
||||
domain_archives = [i.id for i in self.domain.archives]
|
||||
if not domain_archives:
|
||||
self.domain.delete()
|
||||
|
||||
|
||||
class Domain(Base, ModelHelper):
|
||||
|
|
@ -159,7 +98,7 @@ class Domain(Base, ModelHelper):
|
|||
domain = Column(String)
|
||||
directory = Column(MediaPathType)
|
||||
|
||||
urls: InstrumentedList = relationship('URL', primaryjoin='Domain.id==URL.domain_id', order_by='URL.latest_datetime')
|
||||
archives: InstrumentedList = relationship('Archive', primaryjoin='Archive.domain_id==Domain.id')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Domain id={self.id} domain={self.domain} directory={self.directory}>'
|
||||
|
|
@ -167,5 +106,4 @@ class Domain(Base, ModelHelper):
|
|||
def delete(self):
|
||||
session = Session.object_session(self)
|
||||
session.execute('DELETE FROM archive WHERE domain_id=:id', dict(id=self.id))
|
||||
session.execute('DELETE FROM url WHERE domain_id=:id', dict(id=self.id))
|
||||
session.query(Domain).filter_by(id=self.id).delete()
|
||||
|
|
|
|||
|
|
@ -1,12 +1,6 @@
|
|||
from sanic_openapi import doc
|
||||
|
||||
|
||||
class RetrieveURLsRequest:
|
||||
limit = doc.Integer()
|
||||
offset = doc.Integer()
|
||||
domain = doc.String()
|
||||
|
||||
|
||||
class ArchiveDict:
|
||||
id = doc.Integer()
|
||||
url_id = doc.Integer()
|
||||
|
|
@ -25,25 +19,6 @@ class DomainDict:
|
|||
domain = doc.String()
|
||||
|
||||
|
||||
class URLDict:
|
||||
id = doc.Integer()
|
||||
url = doc.String()
|
||||
latest_id = doc.Integer()
|
||||
latest_datetime = doc.DateTime()
|
||||
domain_id = doc.Integer()
|
||||
domain = DomainDict
|
||||
latest = ArchiveDict
|
||||
archives = doc.List(ArchiveDict)
|
||||
|
||||
|
||||
class RetrieveURLsResponse:
|
||||
urls = doc.List(URLDict)
|
||||
|
||||
|
||||
class PostArchiveRequest:
|
||||
url = doc.String(required=True)
|
||||
|
||||
|
||||
class ArchiveSearchRequest:
|
||||
search_str = doc.String()
|
||||
domain = doc.String()
|
||||
|
|
|
|||
|
|
@ -4,12 +4,11 @@ from datetime import datetime
|
|||
|
||||
import mock
|
||||
|
||||
from modules.archive.lib import new_archive, get_or_create_domain_and_url, get_urls, get_url_count, _refresh_archives, \
|
||||
get_new_archive_files, delete_archive
|
||||
from modules.archive.models import Archive, URL, Domain
|
||||
from modules.archive.lib import get_or_create_domain, _refresh_archives, \
|
||||
get_new_archive_files, delete_archive, do_archive, get_domains
|
||||
from modules.archive.models import Archive, Domain
|
||||
from wrolpi.common import get_media_directory
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import InvalidDomain
|
||||
from wrolpi.media_path import MediaPath
|
||||
from wrolpi.root_api import CustomJSONEncoder
|
||||
from wrolpi.test.common import TestAPI, wrap_test_db, wrap_media_directory
|
||||
|
|
@ -55,7 +54,7 @@ class TestArchive(TestAPI):
|
|||
@wrap_test_db
|
||||
def test_new_archive(self):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
archive1 = new_archive('https://example.com', sync=True)
|
||||
archive1 = do_archive('https://example.com')
|
||||
# Everything is filled out.
|
||||
self.assertIsInstance(archive1, Archive)
|
||||
self.assertIsNotNone(archive1.archive_datetime)
|
||||
|
|
@ -77,15 +76,14 @@ class TestArchive(TestAPI):
|
|||
with open(archive1.readability_json_path.path) as fh:
|
||||
self.assertEqual(json.load(fh), {'title': 'ジにてこちら', 'url': 'https://example.com'})
|
||||
|
||||
archive2 = new_archive('https://example.com')
|
||||
# URL and Domain are reused.
|
||||
self.assertEqual(archive1.url, archive2.url)
|
||||
archive2 = do_archive('https://example.com')
|
||||
# Domain is reused.
|
||||
self.assertEqual(archive1.domain, archive2.domain)
|
||||
|
||||
@wrap_test_db
|
||||
def test_no_screenshot(self):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)):
|
||||
archive = new_archive('https://example.com', sync=True)
|
||||
archive = do_archive('https://example.com')
|
||||
self.assertIsInstance(archive.singlefile_path, MediaPath)
|
||||
self.assertIsInstance(archive.readability_path, MediaPath)
|
||||
# Screenshot was empty
|
||||
|
|
@ -94,7 +92,7 @@ class TestArchive(TestAPI):
|
|||
@wrap_test_db
|
||||
def test_no_readability(self):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(readability=False)):
|
||||
archive = new_archive('https://example.com', sync=True)
|
||||
archive = do_archive('https://example.com')
|
||||
self.assertIsInstance(archive.singlefile_path, MediaPath)
|
||||
self.assertIsInstance(archive.screenshot_path, MediaPath)
|
||||
# Readability empty
|
||||
|
|
@ -104,115 +102,41 @@ class TestArchive(TestAPI):
|
|||
@wrap_test_db
|
||||
def test_dict(self):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
d = new_archive('https://example.com', sync=True).dict()
|
||||
d = do_archive('https://example.com').dict()
|
||||
self.assertIsInstance(d, dict)
|
||||
json.dumps(d, cls=CustomJSONEncoder)
|
||||
|
||||
@wrap_test_db
|
||||
def test_relationships(self):
|
||||
with get_db_session(commit=True) as session:
|
||||
domain, url = get_or_create_domain_and_url(session, 'https://wrolpi.org:443')
|
||||
url = 'https://wrolpi.org:443'
|
||||
domain = get_or_create_domain(session, url)
|
||||
archive = Archive(
|
||||
singlefile_path=f'{self.tmp_dir.name}/wrolpi.org:443/foo',
|
||||
title='bar',
|
||||
url_id=url.id,
|
||||
url=url,
|
||||
domain_id=domain.id,
|
||||
)
|
||||
session.add(archive)
|
||||
session.flush()
|
||||
|
||||
url.latest_id = archive.id
|
||||
|
||||
self.assertEqual(archive.domain, domain)
|
||||
self.assertEqual(archive.url, url)
|
||||
|
||||
# Relationships are added in the dict() method.
|
||||
self.assertDictContains(
|
||||
url.dict(),
|
||||
dict(
|
||||
id=1,
|
||||
url='https://wrolpi.org:443',
|
||||
latest_id=1,
|
||||
latest=dict(singlefile_path=self.tmp_path / 'wrolpi.org:443/foo'),
|
||||
domain_id=1,
|
||||
domain=dict(directory=self.tmp_path / 'archive/wrolpi.org:443', domain='wrolpi.org:443'),
|
||||
)
|
||||
)
|
||||
|
||||
@wrap_test_db
|
||||
def test_get_urls(self):
|
||||
# No URLs yet.
|
||||
urls = get_urls()
|
||||
self.assertEqual([], urls)
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(readability=False)):
|
||||
# One set of duplicate URLs
|
||||
new_archive('https://wrolpi.org/one', sync=True)
|
||||
new_archive('https://wrolpi.org/one', sync=True)
|
||||
|
||||
# Unique URLs
|
||||
new_archive('https://wrolpi.org/two', sync=True)
|
||||
new_archive('https://wrolpi.org/three', sync=True)
|
||||
new_archive('https://example.com/one', sync=True)
|
||||
|
||||
urls = get_urls()
|
||||
# There are only 4 because one set is duplicate.
|
||||
self.assertEqual(len(urls), 4)
|
||||
self.assertEqual(
|
||||
['https://example.com/one', 'https://wrolpi.org/three',
|
||||
'https://wrolpi.org/two', 'https://wrolpi.org/one'],
|
||||
[i['url'] for i in urls],
|
||||
)
|
||||
|
||||
# Only one URL for this domain.
|
||||
urls = get_urls(domain='example.com')
|
||||
self.assertEqual(len(urls), 1)
|
||||
self.assertEqual(urls[0]['url'], 'https://example.com/one')
|
||||
|
||||
# Bad domain requested.
|
||||
self.assertRaises(InvalidDomain, get_urls, domain='bad_domain.com')
|
||||
|
||||
# Limit to 3, but with an offset of 2 there are only 2.
|
||||
urls = get_urls(3, 2)
|
||||
self.assertEqual(['https://wrolpi.org/two', 'https://wrolpi.org/one'], [i['url'] for i in urls])
|
||||
|
||||
# First two of this domain.
|
||||
urls = get_urls(2, 0, 'wrolpi.org')
|
||||
self.assertEqual(['https://wrolpi.org/two', 'https://wrolpi.org/one'], [i['url'] for i in urls])
|
||||
# Last two of this domain, but there is only 1.
|
||||
urls = get_urls(2, 2, 'wrolpi.org')
|
||||
self.assertEqual(['https://wrolpi.org/three'], [i['url'] for i in urls])
|
||||
|
||||
@wrap_test_db
|
||||
def test_validate_paths(self):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
archive = new_archive('https://example.com')
|
||||
archive = do_archive('https://example.com')
|
||||
try:
|
||||
with get_db_session(commit=True):
|
||||
archive.singlefile_path = 'asdf'
|
||||
except ValueError as e:
|
||||
self.assertIn('relative', str(e), f'Relative path error was not raised')
|
||||
|
||||
@wrap_test_db
|
||||
def test_get_url_count(self):
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
self.assertRaises(InvalidDomain, get_url_count, 'bad domain')
|
||||
|
||||
self.assertEqual(get_url_count(), 0)
|
||||
new_archive('https://example.com', sync=True)
|
||||
self.assertEqual(get_url_count(), 1)
|
||||
new_archive('https://example.com', sync=True)
|
||||
self.assertEqual(get_url_count(), 1)
|
||||
new_archive('https://example.org', sync=True)
|
||||
self.assertEqual(get_url_count(), 2)
|
||||
new_archive('https://example.org', sync=True)
|
||||
self.assertEqual(get_url_count('example.org'), 1)
|
||||
|
||||
@wrap_test_db
|
||||
def test_get_title_from_html(self):
|
||||
with wrap_media_directory():
|
||||
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
|
||||
archive = new_archive('example.com', sync=True)
|
||||
archive = do_archive('example.com')
|
||||
self.assertEqual(archive.title, 'ジにてこちら')
|
||||
|
||||
def fake_request_archive(_):
|
||||
|
|
@ -225,7 +149,7 @@ class TestArchive(TestAPI):
|
|||
return singlefile, r, s
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive = new_archive('example.com', sync=True)
|
||||
archive = do_archive('example.com')
|
||||
self.assertEqual(archive.title, 'some title')
|
||||
|
||||
def fake_request_archive(_):
|
||||
|
|
@ -238,16 +162,12 @@ class TestArchive(TestAPI):
|
|||
return singlefile, r, s
|
||||
|
||||
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
|
||||
archive = new_archive('example.com', sync=True)
|
||||
archive = do_archive('example.com')
|
||||
self.assertIsNone(archive.title)
|
||||
|
||||
@wrap_test_db
|
||||
def test_refresh_archives(self):
|
||||
with wrap_media_directory():
|
||||
with get_db_session() as session:
|
||||
urls = session.query(URL).count()
|
||||
self.assertEqual(urls, 0)
|
||||
|
||||
archive_directory = get_media_directory() / 'archive'
|
||||
archive_directory.mkdir()
|
||||
|
||||
|
|
@ -274,25 +194,17 @@ class TestArchive(TestAPI):
|
|||
# This single archive is found.
|
||||
_refresh_archives()
|
||||
with get_db_session() as session:
|
||||
self.assertEqual(session.query(URL).count(), 1)
|
||||
self.assertEqual(session.query(Archive).count(), 1)
|
||||
|
||||
url: URL = session.query(URL).one()
|
||||
archive: Archive = session.query(Archive).one()
|
||||
self.assertEqual(url.latest, archive)
|
||||
# latest_datetime is set, the timestamp in the file name is assumed to be UTC.
|
||||
self.assertEqual(str(url.latest_datetime), '2021-10-05 16:20:10.346823-06:00')
|
||||
|
||||
# Running the refresh does not result in a new archive.
|
||||
_refresh_archives()
|
||||
with get_db_session() as session:
|
||||
self.assertEqual(session.query(URL).count(), 1)
|
||||
self.assertEqual(session.query(Archive).count(), 1)
|
||||
|
||||
|
||||
def test_archive_refresh_deleted_archive(test_session, archive_directory, archive_factory):
|
||||
"""
|
||||
Archives/URLs/Domains should be deleted when archive files are deleted.
|
||||
Archives/Domains should be deleted when archive files are deleted.
|
||||
"""
|
||||
archive1 = archive_factory('example.com', 'https://example.com/1')
|
||||
archive2 = archive_factory('example.com', 'https://example.com/1')
|
||||
|
|
@ -304,33 +216,32 @@ def test_archive_refresh_deleted_archive(test_session, archive_directory, archiv
|
|||
empty = archive_directory / 'empty'
|
||||
empty.mkdir()
|
||||
|
||||
def check_counts(archive_count, url_count, domain_count):
|
||||
def check_counts(archive_count, domain_count):
|
||||
assert test_session.query(Archive).count() == archive_count
|
||||
assert test_session.query(URL).count() == url_count
|
||||
assert test_session.query(Domain).count() == domain_count
|
||||
|
||||
# All 3 archives are already in the DB.
|
||||
check_counts(archive_count=5, url_count=4, domain_count=3)
|
||||
check_counts(archive_count=5, domain_count=3)
|
||||
_refresh_archives()
|
||||
check_counts(archive_count=5, url_count=4, domain_count=3)
|
||||
check_counts(archive_count=5, domain_count=3)
|
||||
|
||||
# Delete archive2's files, it's the latest for 'https://example.com/1'
|
||||
for path in archive2.my_paths():
|
||||
path.unlink()
|
||||
_refresh_archives()
|
||||
check_counts(archive_count=4, url_count=4, domain_count=3)
|
||||
check_counts(archive_count=4, domain_count=3)
|
||||
|
||||
# Delete archive1's files, now the URL is empty.
|
||||
for path in archive1.my_paths():
|
||||
path.unlink()
|
||||
_refresh_archives()
|
||||
check_counts(archive_count=3, url_count=3, domain_count=3)
|
||||
check_counts(archive_count=3, domain_count=3)
|
||||
|
||||
# Delete archive3, now there is now example.com domain
|
||||
for path in archive3.my_paths():
|
||||
path.unlink()
|
||||
_refresh_archives()
|
||||
check_counts(archive_count=2, url_count=2, domain_count=2)
|
||||
check_counts(archive_count=2, domain_count=2)
|
||||
|
||||
# Delete all the rest of the archives
|
||||
for path in archive4.my_paths():
|
||||
|
|
@ -338,7 +249,7 @@ def test_archive_refresh_deleted_archive(test_session, archive_directory, archiv
|
|||
for path in archive5.my_paths():
|
||||
path.unlink()
|
||||
_refresh_archives()
|
||||
check_counts(archive_count=0, url_count=0, domain_count=0)
|
||||
check_counts(archive_count=0, domain_count=0)
|
||||
|
||||
|
||||
def test_refresh_archives_fills_contents(test_session, archive_factory, test_client):
|
||||
|
|
@ -380,18 +291,36 @@ def test_delete_archive(test_session, archive_factory):
|
|||
archive2 = archive_factory('example.com', 'https://example.com/1')
|
||||
archive3 = archive_factory('example.com', 'https://example.com/1')
|
||||
|
||||
url = test_session.query(URL).one()
|
||||
assert url.latest == archive3
|
||||
|
||||
# Delete the oldest.
|
||||
delete_archive(archive1.id)
|
||||
|
||||
# Delete the latest.
|
||||
delete_archive(archive3.id)
|
||||
|
||||
# Delete the last archive. The URL and Domain should also be deleted.
|
||||
# Delete the last archive. The Domain should also be deleted.
|
||||
delete_archive(archive2.id)
|
||||
url = test_session.query(URL).one_or_none()
|
||||
assert url is None
|
||||
domain = test_session.query(Domain).one_or_none()
|
||||
assert domain is None
|
||||
|
||||
|
||||
def test_get_domains(test_session, archive_factory):
|
||||
"""
|
||||
`get_domains` gets only Domains with Archives.
|
||||
"""
|
||||
archive1 = archive_factory('example.com')
|
||||
archive2 = archive_factory('example.com')
|
||||
archive3 = archive_factory('example.org')
|
||||
|
||||
assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org']
|
||||
|
||||
archive2.delete()
|
||||
test_session.commit()
|
||||
assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org']
|
||||
|
||||
archive1.delete()
|
||||
test_session.commit()
|
||||
assert [i['domain'] for i in get_domains()] == ['example.org']
|
||||
|
||||
archive3.delete()
|
||||
test_session.commit()
|
||||
assert [i['domain'] for i in get_domains()] == []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue