mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Moving data from models to FileGroup.
* Ebook.creator -> FileGroup.author * Video.viewed -> FileGroup.viewed * Video.duration -> FileGroup.length * Video.url -> FileGroup.url * Archive.url -> FileGroup.url
This commit is contained in:
parent
ae9d188391
commit
15687e534a
32 changed files with 418 additions and 209 deletions
49
alembic/versions/365c985893b8_ebook_creator.py
Normal file
49
alembic/versions/365c985893b8_ebook_creator.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""eBook creator.
|
||||
|
||||
Revision ID: 365c985893b8
|
||||
Revises: 5ad9efbf3570
|
||||
Create Date: 2023-10-29 16:54:28.319427
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '365c985893b8'
|
||||
down_revision = '5ad9efbf3570'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('''
|
||||
UPDATE file_group
|
||||
SET author=creator
|
||||
FROM ebook
|
||||
WHERE ebook.file_group_id = file_group.id
|
||||
''')
|
||||
session.execute('DROP INDEX IF EXISTS ebook_creator_idx')
|
||||
session.execute('ALTER TABLE ebook DROP COLUMN IF EXISTS creator')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('ALTER TABLE ebook ADD COLUMN IF NOT EXISTS creator TEXT')
|
||||
session.execute('''
|
||||
UPDATE ebook
|
||||
SET creator=author
|
||||
FROM file_group
|
||||
WHERE
|
||||
ebook.file_group_id = file_group.id
|
||||
AND model = 'ebook'
|
||||
''')
|
||||
session.execute('CREATE INDEX IF NOT EXISTS ebook_creator_idx ON ebook(creator)')
|
||||
141
alembic/versions/5ad9efbf3570_filegroup_length.py
Normal file
141
alembic/versions/5ad9efbf3570_filegroup_length.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
"""
|
||||
Migrates Video.duration/.url/.viewed/Archive.url to FileGroup.length/.url/.viewed.
|
||||
|
||||
Revision ID: 5ad9efbf3570
|
||||
Revises: 4b54ad052e2e
|
||||
Create Date: 2023-10-24 09:25:33.818373
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '5ad9efbf3570'
|
||||
down_revision = '4b54ad052e2e'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('ALTER TABLE file_group ADD COLUMN length INT DEFAULT 0')
|
||||
session.execute('''
|
||||
UPDATE file_group
|
||||
SET length=video.duration
|
||||
FROM video
|
||||
WHERE file_group.id = video.file_group_id
|
||||
''')
|
||||
|
||||
# Migrate Archive.url and Video.url to FileGroup.url.
|
||||
session.execute('ALTER TABLE file_group ADD COLUMN url TEXT')
|
||||
session.execute('''
|
||||
UPDATE file_group
|
||||
SET url=archive.url
|
||||
FROM archive
|
||||
WHERE file_group.id = archive.file_group_id
|
||||
''')
|
||||
session.execute('''
|
||||
UPDATE file_group
|
||||
SET url=video.url
|
||||
FROM video
|
||||
WHERE file_group.id = video.file_group_id
|
||||
''')
|
||||
|
||||
# Migrate Archive.archive_datetime to FileGroup.download_datetime.
|
||||
session.execute('ALTER TABLE file_group ADD COLUMN download_datetime TIMESTAMP WITH TIME ZONE')
|
||||
session.execute('''
|
||||
UPDATE file_group
|
||||
SET download_datetime=archive.archive_datetime
|
||||
FROM archive
|
||||
WHERE file_group.id = archive.file_group_id
|
||||
''')
|
||||
|
||||
# Migrate Video.viewed to FileGroup.viewed.
|
||||
session.execute('ALTER TABLE file_group ADD COLUMN viewed TIMESTAMP WITH TIME ZONE')
|
||||
session.execute('''
|
||||
UPDATE file_group
|
||||
SET viewed=video.viewed
|
||||
FROM video
|
||||
WHERE file_group.id = video.file_group_id
|
||||
''')
|
||||
|
||||
session.execute('DROP INDEX IF EXISTS archive_archive_datetime_idx')
|
||||
session.execute('DROP INDEX IF EXISTS archive_url_idx')
|
||||
session.execute('DROP INDEX IF EXISTS video_url_idx')
|
||||
session.execute('DROP INDEX IF EXISTS video_viewed_idx')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS archive_datetime')
|
||||
session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS url')
|
||||
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS url')
|
||||
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS viewed')
|
||||
|
||||
session.execute('CREATE INDEX file_group_download_datetime_idx ON file_group(download_datetime)')
|
||||
session.execute('CREATE INDEX file_group_length_idx ON file_group(length)')
|
||||
session.execute('CREATE INDEX file_group_url_idx ON file_group(url)')
|
||||
session.execute('CREATE INDEX file_group_viewed_idx ON file_group(viewed)')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS duration INTEGER')
|
||||
session.execute('''
|
||||
UPDATE video
|
||||
SET duration=file_group.length
|
||||
FROM file_group
|
||||
WHERE file_group.id = video.file_group_id
|
||||
''')
|
||||
|
||||
session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS url TEXT')
|
||||
session.execute('''
|
||||
UPDATE video
|
||||
SET url=file_group.url
|
||||
FROM file_group
|
||||
WHERE file_group.id = video.file_group_id
|
||||
''')
|
||||
|
||||
# Video has `viewed`, Archive does not.
|
||||
session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS viewed TIMESTAMP WITH TIME ZONE')
|
||||
session.execute('''
|
||||
UPDATE video
|
||||
SET url=file_group.viewed
|
||||
FROM file_group
|
||||
WHERE file_group.id = video.file_group_id
|
||||
''')
|
||||
|
||||
session.execute('ALTER TABLE archive ADD COLUMN IF NOT EXISTS url TEXT')
|
||||
session.execute('''
|
||||
UPDATE archive
|
||||
SET url=file_group.url
|
||||
FROM file_group
|
||||
WHERE file_group.id = archive.file_group_id
|
||||
''')
|
||||
|
||||
session.execute('ALTER TABLE archive ADD COLUMN IF NOT EXISTS archive_datetime TIMESTAMP WITH TIME ZONE')
|
||||
session.execute('''
|
||||
UPDATE archive
|
||||
SET archive_datetime=file_group.download_datetime
|
||||
FROM file_group
|
||||
WHERE file_group.id = archive.file_group_id
|
||||
''')
|
||||
|
||||
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS download_datetime')
|
||||
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS length')
|
||||
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS url')
|
||||
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS viewed')
|
||||
session.execute('DROP INDEX IF EXISTS file_group_download_datetime_idx')
|
||||
session.execute('DROP INDEX IF EXISTS file_group_length_idx')
|
||||
session.execute('DROP INDEX IF EXISTS file_group_url_idx')
|
||||
session.execute('DROP INDEX IF EXISTS file_group_viewed_idx')
|
||||
|
||||
session.execute('CREATE INDEX IF NOT EXISTS archive_archive_datetime_idx ON archive(archive_datetime)')
|
||||
session.execute('CREATE INDEX IF NOT EXISTS archive_url_idx ON archive(url)')
|
||||
session.execute('CREATE INDEX IF NOT EXISTS video_duration_idx ON video(duration)')
|
||||
session.execute('CREATE INDEX IF NOT EXISTS video_url_idx ON video(url)')
|
||||
session.execute('CREATE INDEX IF NOT EXISTS video_viewed_idx ON video(viewed)')
|
||||
|
|
@ -25,6 +25,15 @@ def upgrade():
|
|||
session.execute('ALTER TABLE file_group ADD COLUMN author TEXT')
|
||||
session.execute('ALTER TABLE file_group ADD COLUMN published_datetime TIMESTAMP WITH TIME ZONE')
|
||||
session.execute('ALTER TABLE file_group ADD COLUMN published_modified_datetime TIMESTAMP WITH TIME ZONE')
|
||||
|
||||
# Migrate Video.upload_date to FileGroup.published_datetime.
|
||||
session.execute('''
|
||||
UPDATE file_group
|
||||
SET published_datetime=upload_date
|
||||
FROM video
|
||||
WHERE file_group.id = video.file_group_id
|
||||
''')
|
||||
|
||||
session.execute('CREATE INDEX IF NOT EXISTS file_group_author_idx ON file_group(author)')
|
||||
session.execute('CREATE INDEX IF NOT EXISTS file_group_published_datetime_idx ON file_group(published_datetime)')
|
||||
session.execute('CREATE INDEX IF NOT EXISTS file_group_published_modified_datetime_idx'
|
||||
|
|
@ -34,9 +43,19 @@ def upgrade():
|
|||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
# Migrate FileGroup.published_datetime to Video.upload_date.
|
||||
session.execute('''
|
||||
UPDATE video
|
||||
SET upload_date=published_datetime
|
||||
FROM file_group
|
||||
WHERE file_group.id = video.file_group_id
|
||||
''')
|
||||
|
||||
session.execute('DROP INDEX IF EXISTS file_group_author_idx')
|
||||
session.execute('DROP INDEX IF EXISTS file_group_published_datetime_idx')
|
||||
session.execute('DROP INDEX IF EXISTS file_group_published_modified_datetime_idx')
|
||||
|
||||
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS author')
|
||||
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_datetime')
|
||||
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_modified_datetime')
|
||||
|
|
|
|||
|
|
@ -69,10 +69,7 @@ function ArchivePage() {
|
|||
const {archiveFile, history, fetchArchive} = useArchive(archiveId);
|
||||
const {theme} = useContext(ThemeContext);
|
||||
|
||||
let title;
|
||||
if (archiveFile && archiveFile.archive) {
|
||||
title = archiveFile.archive.title;
|
||||
}
|
||||
let title = archiveFile ? archiveFile.title ? archiveFile.title : archiveFile.name : null;
|
||||
useTitle(title);
|
||||
|
||||
if (archiveFile === null) {
|
||||
|
|
@ -168,7 +165,7 @@ function ArchivePage() {
|
|||
await fetchArchive();
|
||||
}
|
||||
|
||||
const archivedDatetimeString = isoDatetimeToString(data.archive_datetime, true);
|
||||
const archivedDatetimeString = isoDatetimeToString(archiveFile.download_datetime, true);
|
||||
const publishedDatetimeString = archiveFile.published_datetime ? isoDatetimeToString(archiveFile.published_datetime, true)
|
||||
: 'unknown';
|
||||
const modifiedDatetimeString = archiveFile.published_modified_datetime
|
||||
|
|
@ -305,7 +302,7 @@ export function ArchiveCard({file}) {
|
|||
</CardLink>}
|
||||
<CardMeta {...s}>
|
||||
<p>
|
||||
{isoDatetimeToString(data.archive_datetime)}
|
||||
{isoDatetimeToString(file.published_datetime)}
|
||||
</p>
|
||||
</CardMeta>
|
||||
<CardDescription>
|
||||
|
|
@ -426,8 +423,11 @@ function ArchivesPage() {
|
|||
} = useSearchArchives();
|
||||
|
||||
let archiveOrders = [
|
||||
{value: 'date', text: 'Date'},
|
||||
{value: 'published_datetime', text: 'Published Date', short: 'P.Date'},
|
||||
{value: 'published_modified_datetime', text: 'Modified Date', short: 'M.Date'},
|
||||
{value: 'download_datetime', text: 'Download Date', short: 'D.Date'},
|
||||
{value: 'size', text: 'Size'},
|
||||
{value: 'viewed', text: 'Recently Viewed', short: 'R.Viewed'},
|
||||
];
|
||||
|
||||
if (searchStr) {
|
||||
|
|
|
|||
|
|
@ -148,27 +148,26 @@ export function isoDatetimeToElapsedPopup(dt) {
|
|||
return secondsToElapsedPopup(d.getTime() / 1000);
|
||||
}
|
||||
|
||||
export function secondsToDuration(video) {
|
||||
let duration = video.duration;
|
||||
let hours = Math.floor(duration / 3600);
|
||||
duration -= hours * 3600;
|
||||
let minutes = Math.floor(duration / 60);
|
||||
let seconds = duration - (minutes * 60);
|
||||
export function secondsToHMS(totalSeconds) {
|
||||
let hours = Math.floor(totalSeconds / 3600);
|
||||
totalSeconds -= hours * 3600;
|
||||
let minutes = Math.floor(totalSeconds / 60);
|
||||
let seconds = totalSeconds - (minutes * 60);
|
||||
|
||||
hours = String('00' + hours).slice(-2);
|
||||
minutes = String('00' + minutes).slice(-2);
|
||||
seconds = String('00' + seconds).slice(-2);
|
||||
|
||||
return [hours, minutes, seconds];
|
||||
return {hours, minutes, seconds};
|
||||
}
|
||||
|
||||
export function Duration({video}) {
|
||||
let [hours, minutes, seconds] = secondsToDuration(video);
|
||||
export function Duration({totalSeconds}) {
|
||||
const {hours, minutes, seconds} = secondsToHMS(totalSeconds);
|
||||
|
||||
if (hours > 0) {
|
||||
return <div className="duration-overlay">{hours}:{minutes}:{seconds}</div>
|
||||
} else if (video.duration) {
|
||||
return <div className="duration-overlay">{minutes}:{seconds}</div>
|
||||
return <div className='duration-overlay'>{hours}:{minutes}:{seconds}</div>
|
||||
} else if (totalSeconds) {
|
||||
return <div className='duration-overlay'>{minutes}:{seconds}</div>
|
||||
} else {
|
||||
return <></>
|
||||
}
|
||||
|
|
@ -224,7 +223,7 @@ export function RequiredAsterisk() {
|
|||
return <span style={{color: '#db2828'}}> *</span>
|
||||
}
|
||||
|
||||
export let defaultVideoOrder = '-upload_date';
|
||||
export let defaultVideoOrder = '-published_datetime';
|
||||
export let defaultSearchOrder = 'rank';
|
||||
|
||||
export const frequencyOptions = [{key: null, text: '', value: null}, {
|
||||
|
|
@ -1277,7 +1276,7 @@ export function SortButton({sorts = []}) {
|
|||
</Modal>
|
||||
<ButtonGroup icon>
|
||||
<Button icon={desc ? 'sort down' : 'sort up'} onClick={() => toggleDesc()}/>
|
||||
<Button content={selectedSort['text']} onClick={() => setOpen(true)}/>
|
||||
<Button content={selectedSort['short'] || selectedSort['text']} onClick={() => setOpen(true)}/>
|
||||
</ButtonGroup>
|
||||
</>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,7 +70,6 @@ import {Headlines} from "./Headline";
|
|||
|
||||
function EbookCard({file}) {
|
||||
const {s} = useContext(ThemeContext);
|
||||
let {data} = file;
|
||||
|
||||
const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
|
||||
const isEpub = file['mimetype'].startsWith('application/epub');
|
||||
|
|
@ -83,12 +82,12 @@ function EbookCard({file}) {
|
|||
<CardHeader>
|
||||
<Container textAlign='left'>
|
||||
<ExternalCardLink to={viewerUrl || downloadUrl}>
|
||||
{cardTitleWrapper(data ? data.title : file.title)}
|
||||
{cardTitleWrapper(file ? file.title : file.title)}
|
||||
</ExternalCardLink>
|
||||
</Container>
|
||||
</CardHeader>
|
||||
<CardMeta>
|
||||
{data.creator ? <b {...s}>{data.creator}</b> : null}
|
||||
{file.author ? <b {...s}>{file.author}</b> : null}
|
||||
{file.size && <p {...s}>{humanFileSize(file.size)}</p>}
|
||||
</CardMeta>
|
||||
</CardContent>
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
const navigate = useNavigate();
|
||||
const videoRef = React.useRef();
|
||||
|
||||
useTitle(videoFile === null || videoFile === undefined ? null : videoFile.video.title);
|
||||
useTitle(videoFile ? videoFile.title ? videoFile.title : videoFile.name : null);
|
||||
let video;
|
||||
if (videoFile) {
|
||||
video = videoFile.video;
|
||||
|
|
@ -241,7 +241,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
<Segment>
|
||||
|
||||
<Header as='h2'>{title}</Header>
|
||||
{video.upload_date && <h3>{isoDatetimeToString(video.upload_date)}</h3>}
|
||||
{videoFile.published_datetime && <h3>{isoDatetimeToString(videoFile.published_datetime)}</h3>}
|
||||
<h3>
|
||||
{channel && <Link to={`/videos/channel/${channel.id}/video`}>
|
||||
{channel.name}
|
||||
|
|
|
|||
|
|
@ -84,10 +84,11 @@ function VideosPage() {
|
|||
useTitle(title);
|
||||
|
||||
let videoOrders = [
|
||||
{value: 'upload_date', text: 'Date'},
|
||||
{value: 'duration', text: 'Duration'},
|
||||
{value: 'published_datetime', text: 'Published Date', short: 'P.Date'},
|
||||
{value: 'length', text: 'Length'},
|
||||
{value: 'size', text: 'Size'},
|
||||
{value: 'view_count', text: 'Views'},
|
||||
{value: 'viewed', text: 'Recently Viewed', short: 'R.Viewed'},
|
||||
]
|
||||
|
||||
if (searchStr) {
|
||||
|
|
@ -291,7 +292,7 @@ export function VideoCard({file}) {
|
|||
// Default to video FilePreview for lone video files.
|
||||
let video_url;
|
||||
|
||||
const upload_date = isoDatetimeToString(video.upload_date);
|
||||
const published_datetime = isoDatetimeToString(file.published_datetime);
|
||||
// A video may not have a channel.
|
||||
const channel = video.channel ? video.channel : null;
|
||||
let channel_url = null;
|
||||
|
|
@ -320,7 +321,7 @@ export function VideoCard({file}) {
|
|||
const color = mimetypeColor(file.mimetype);
|
||||
return <Card color={color}>
|
||||
{poster}
|
||||
<Duration video={video}/>
|
||||
<Duration totalSeconds={file.length}/>
|
||||
<CardContent {...s}>
|
||||
<CardHeader>
|
||||
<Container textAlign='left'>
|
||||
|
|
@ -332,7 +333,7 @@ export function VideoCard({file}) {
|
|||
{channel && <Link to={channel_url} className="no-link-underscore card-link">
|
||||
<b {...s}>{channel.name}</b>
|
||||
</Link>}
|
||||
<p {...s}>{upload_date}</p>
|
||||
<p {...s}>{published_datetime}</p>
|
||||
</Container>
|
||||
</CardDescription>
|
||||
</CardContent>
|
||||
|
|
|
|||
|
|
@ -61,8 +61,8 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
|
||||
@optional_session
|
||||
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
|
||||
archives = list(session.query(Archive).filter(Archive.url.in_(urls)))
|
||||
return archives
|
||||
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
|
||||
return file_groups
|
||||
|
||||
async def do_singlefile(self, download: Download) -> bytes:
|
||||
"""Create a Singlefile from the archive's URL."""
|
||||
|
|
@ -226,14 +226,12 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
|||
file_group.d_text = contents
|
||||
file_group.data = {
|
||||
'id': archive.id,
|
||||
'archive_datetime': archive.archive_datetime,
|
||||
'domain': archive.domain.domain if archive.domain else None,
|
||||
'readability_json_path': archive.readability_json_path,
|
||||
'readability_path': archive.readability_path,
|
||||
'readability_txt_path': archive.readability_txt_path,
|
||||
'screenshot_path': archive.screenshot_path,
|
||||
'singlefile_path': archive.singlefile_path,
|
||||
'url': archive.url,
|
||||
}
|
||||
|
||||
return archive
|
||||
|
|
|
|||
|
|
@ -40,8 +40,8 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
domain_dir = archive_directory / domain_dir
|
||||
domain_dir.mkdir(exist_ok=True)
|
||||
|
||||
archive_datetime = next(now)
|
||||
timestamp = archive_strftime(archive_datetime)
|
||||
download_datetime = next(now)
|
||||
timestamp = archive_strftime(download_datetime)
|
||||
|
||||
json_contents = {}
|
||||
if title != 'NA':
|
||||
|
|
@ -76,7 +76,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
archive = Archive.from_paths(test_session, *files)
|
||||
archive.url = url
|
||||
archive.title = title
|
||||
archive.archive_datetime = archive.file_group.published_datetime = next(now)
|
||||
archive.file_group.download_datetime = archive.file_group.published_datetime = next(now)
|
||||
archive.file_group.modification_datetime = next(now)
|
||||
archive.domain = domain
|
||||
archive.validate()
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
|
|||
|
||||
with get_db_session(commit=True) as session:
|
||||
archive = Archive.from_paths(session, *paths)
|
||||
archive.archive_datetime = now()
|
||||
archive.file_group.download_datetime = now()
|
||||
archive.url = url
|
||||
archive.domain = get_or_create_domain(session, url)
|
||||
session.flush()
|
||||
|
|
@ -378,6 +378,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
|
|||
def get_archive(session, archive_id: int) -> Archive:
|
||||
"""Get an Archive."""
|
||||
archive = Archive.find_by_id(archive_id, session=session)
|
||||
archive.file_group.set_viewed()
|
||||
return archive
|
||||
|
||||
|
||||
|
|
@ -439,20 +440,38 @@ def get_domains():
|
|||
|
||||
|
||||
ARCHIVE_ORDERS = {
|
||||
'date': (date := 'fg.published_datetime ASC, a.archive_datetime ASC'),
|
||||
'-date': (_date := 'fg.published_datetime DESC NULLS LAST, a.archive_datetime DESC NULLS LAST'),
|
||||
'published_datetime': (date := 'fg.published_datetime ASC, fg.download_datetime ASC'),
|
||||
'-published_datetime': (_date := 'fg.published_datetime DESC NULLS LAST, fg.download_datetime DESC NULLS LAST'),
|
||||
'published_modified_datetime': f'fg.published_modified_datetime ASC NULLS LAST, {date}',
|
||||
'-published_modified_datetime': f'fg.published_modified_datetime DESC, {_date}',
|
||||
'download_datetime': 'fg.download_datetime ASC',
|
||||
'-download_datetime': 'fg.download_datetime DESC',
|
||||
'rank': f'2 DESC, {_date}',
|
||||
'-rank': f'2 ASC, {date}',
|
||||
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
|
||||
'-size': 'fg.size DESC NULLS LAST, LOWER(fg.primary_path) DESC',
|
||||
'viewed': 'fg.viewed ASC',
|
||||
'-viewed': 'fg.viewed DESC',
|
||||
}
|
||||
ORDER_GROUP_BYS = {
|
||||
'date': 'fg.published_datetime, a.archive_datetime',
|
||||
'-date': 'fg.published_datetime, a.archive_datetime',
|
||||
'rank': 'fg.published_datetime, a.archive_datetime',
|
||||
'-rank': 'fg.published_datetime, a.archive_datetime',
|
||||
'published_datetime': 'fg.published_datetime, fg.download_datetime',
|
||||
'-published_datetime': 'fg.published_datetime, fg.download_datetime',
|
||||
'published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime',
|
||||
'-published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime',
|
||||
'download_datetime': 'fg.download_datetime',
|
||||
'-download_datetime': 'fg.download_datetime',
|
||||
'rank': 'fg.published_datetime, fg.download_datetime',
|
||||
'-rank': 'fg.published_datetime, fg.download_datetime',
|
||||
'size': 'fg.size, fg.published_datetime, fg.primary_path',
|
||||
'-size': 'fg.size, fg.published_datetime, fg.primary_path',
|
||||
'viewed': 'fg.viewed',
|
||||
'-viewed': 'fg.viewed',
|
||||
}
|
||||
NO_NULL_ORDERS = {
|
||||
'viewed': 'fg.viewed IS NOT NULL',
|
||||
'-viewed': 'fg.viewed IS NOT NULL',
|
||||
'published_modified_datetime': 'fg.published_modified_datetime IS NOT NULL',
|
||||
'-published_modified_datetime': 'fg.published_modified_datetime IS NOT NULL',
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -461,10 +480,10 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
-> Tuple[List[dict], int]:
|
||||
# Always filter FileGroups to Archives.
|
||||
wheres = []
|
||||
group_by = 'fg.published_datetime, a.archive_datetime, a.file_group_id'
|
||||
group_by = 'fg.published_datetime, fg.download_datetime, a.file_group_id'
|
||||
|
||||
params = dict(search_str=search_str, offset=int(offset), limit=int(limit))
|
||||
order_by = ARCHIVE_ORDERS['-date']
|
||||
order_by = ARCHIVE_ORDERS['-published_datetime']
|
||||
|
||||
select_columns = ''
|
||||
if search_str:
|
||||
|
|
@ -480,7 +499,9 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
order_by = ARCHIVE_ORDERS[order]
|
||||
group_by = f'{group_by}, {ORDER_GROUP_BYS[order]}'
|
||||
except KeyError:
|
||||
raise InvalidOrderBy(f'Invalid order by: {order}')
|
||||
raise InvalidOrderBy(f'Invalid order byy {order}')
|
||||
if order in NO_NULL_ORDERS:
|
||||
wheres.append(NO_NULL_ORDERS[order])
|
||||
|
||||
if tag_names:
|
||||
# Filter all FileGroups by those that have been tagged with the provided tag names.
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from sqlalchemy.orm.collections import InstrumentedList
|
|||
|
||||
from wrolpi import dates
|
||||
from wrolpi.common import ModelHelper, Base, logger
|
||||
from wrolpi.dates import TZDateTime
|
||||
from wrolpi.db import optional_session
|
||||
from wrolpi.errors import UnknownArchive
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -31,9 +30,6 @@ class Archive(Base, ModelHelper):
|
|||
__tablename__ = 'archive'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
archive_datetime = Column(TZDateTime)
|
||||
url = Column(String)
|
||||
|
||||
domain_id = Column(Integer, ForeignKey('domains.id'))
|
||||
domain = relationship('Domain', primaryjoin='Archive.domain_id==Domain.id')
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
|
||||
|
|
@ -41,20 +37,20 @@ class Archive(Base, ModelHelper):
|
|||
|
||||
def __repr__(self):
|
||||
if self.domain:
|
||||
return f'<Archive id={self.id} url={self.url} singlefile={repr(str(self.singlefile_path))} ' \
|
||||
return f'<Archive id={self.id} url={self.file_group.url} singlefile={repr(str(self.singlefile_path))} ' \
|
||||
f'domain={self.domain.domain}>'
|
||||
return f'<Archive id={self.id} url={self.url} singlefile={repr(str(self.singlefile_path))}>'
|
||||
return f'<Archive id={self.id} url={self.file_group.url} singlefile={repr(str(self.singlefile_path))}>'
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
|
||||
def get_by_id(id_: int, session: Session = None) -> Optional['Archive']:
|
||||
"""Attempt to find an Archive with the provided id. Returns None if it cannot be found."""
|
||||
archive = session.query(Archive).filter_by(id=id_).one_or_none()
|
||||
return archive
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def find_by_id(id_: int, session: Session = None) -> Base:
|
||||
def find_by_id(id_: int, session: Session = None) -> 'Archive':
|
||||
"""Find an Archive with the provided id, raises an exception when no Archive is found.
|
||||
|
||||
@raise UnknownArchive: if the Archive can not be found"""
|
||||
|
|
@ -84,7 +80,7 @@ class Archive(Base, ModelHelper):
|
|||
if not isinstance(other, Archive):
|
||||
raise ValueError(f'Cannot compare {type(other)} to Archive!')
|
||||
|
||||
return self.archive_datetime > other.archive_datetime
|
||||
return self.file_group.published_datetime > other.file_group.published_datetime
|
||||
|
||||
def delete(self):
|
||||
self.file_group.delete()
|
||||
|
|
@ -105,9 +101,11 @@ class Archive(Base, ModelHelper):
|
|||
session = Session.object_session(self)
|
||||
history = list(session.query(Archive).filter(
|
||||
Archive.id != self.id,
|
||||
Archive.url == self.url,
|
||||
Archive.url != None, # noqa
|
||||
).order_by(Archive.archive_datetime))
|
||||
FileGroup.url == self.file_group.url,
|
||||
FileGroup.url != None, # noqa
|
||||
) \
|
||||
.join(FileGroup, FileGroup.id == Archive.file_group_id) \
|
||||
.order_by(FileGroup.published_datetime))
|
||||
return history
|
||||
|
||||
@property
|
||||
|
|
@ -192,7 +190,7 @@ class Archive(Base, ModelHelper):
|
|||
raise InvalidArchive() from e
|
||||
|
||||
# Readability is most trusted, it should overwrite any previous data.
|
||||
self.url = url or self.url
|
||||
self.file_group.url = url or self.file_group.url
|
||||
self.file_group.title = title or self.file_group.title
|
||||
|
||||
def apply_singlefile_data(self):
|
||||
|
|
@ -202,7 +200,7 @@ class Archive(Base, ModelHelper):
|
|||
# Can't read contents of nothing.
|
||||
raise ValueError('Cannot read singlefile data when this has no files')
|
||||
|
||||
if self.url and self.archive_datetime:
|
||||
if self.file_group.url and self.file_group.published_datetime:
|
||||
# This data has already been read.
|
||||
return
|
||||
|
||||
|
|
@ -212,21 +210,21 @@ class Archive(Base, ModelHelper):
|
|||
logger.error(f'Could not find SingleFile header in {self}')
|
||||
return
|
||||
|
||||
if not self.url:
|
||||
if not self.file_group.url:
|
||||
try:
|
||||
if match := MATCH_URL.findall(head):
|
||||
self.url = match[0].strip()
|
||||
self.file_group.url = match[0].strip()
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
|
||||
|
||||
if not self.archive_datetime:
|
||||
if not self.file_group.download_datetime:
|
||||
try:
|
||||
if match := MATCH_DATE.findall(head):
|
||||
dt = match[0].strip()
|
||||
dt = ' '.join(dt.split(' ')[:5])
|
||||
# SingleFile uses GMT.
|
||||
dt = dates.strpdate(dt).replace(tzinfo=pytz.timezone('GMT'))
|
||||
self.archive_datetime = dt
|
||||
self.file_group.download_datetime = dt
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
|
||||
|
||||
|
|
@ -234,11 +232,11 @@ class Archive(Base, ModelHelper):
|
|||
"""Get the domain from the URL."""
|
||||
from modules.archive.lib import get_or_create_domain
|
||||
domain = None
|
||||
if self.url:
|
||||
if self.file_group.url:
|
||||
session = Session.object_session(self)
|
||||
if not session:
|
||||
raise ValueError('No session found!')
|
||||
domain = get_or_create_domain(session, self.url)
|
||||
domain = get_or_create_domain(session, self.file_group.url)
|
||||
# Clear domain if the URL is missing.
|
||||
self.domain_id = domain.id if domain else None
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ class ArchiveDict:
|
|||
readability_txt_path: str
|
||||
screenshot_path: str
|
||||
title: str
|
||||
archive_datetime: datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -88,37 +88,33 @@ async def test_archive_title(test_session, archive_factory, singlefile_contents_
|
|||
title='json title',
|
||||
singlefile_contents=singlefile_contents_factory('singlefile title', 'https://example.com/singlefile-url'),
|
||||
)
|
||||
test_session.query(FileGroup).delete()
|
||||
test_session.commit()
|
||||
|
||||
await files_lib.refresh_files()
|
||||
async def reset_and_get_archive():
|
||||
test_session.query(FileGroup).delete()
|
||||
test_session.commit()
|
||||
await files_lib.refresh_files()
|
||||
return test_session.query(Archive).one()
|
||||
|
||||
archive1: Archive = await reset_and_get_archive()
|
||||
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
assert test_session.query(Archive).count() == 1
|
||||
|
||||
archive1: Archive = test_session.query(Archive).one()
|
||||
|
||||
# Readability JSON is trusted first.
|
||||
assert archive1.url == 'https://example.com/json-url'
|
||||
assert archive1.file_group.url == 'https://example.com/json-url'
|
||||
assert archive1.file_group.title == 'json title'
|
||||
|
||||
# Delete JSON file.
|
||||
archive1.readability_json_path.unlink()
|
||||
test_session.delete(archive1)
|
||||
await files_lib.refresh_files()
|
||||
|
||||
archive1: Archive = test_session.query(Archive).one()
|
||||
assert archive1.url == 'https://example.com/singlefile-url'
|
||||
archive1: Archive = await reset_and_get_archive()
|
||||
assert archive1.file_group.url == 'https://example.com/singlefile-url'
|
||||
assert archive1.file_group.title == 'singlefile title'
|
||||
|
||||
# Clear out title and URL from Singlefile.
|
||||
archive1.singlefile_path.write_text(singlefile_contents_factory('', ''))
|
||||
test_session.delete(archive1)
|
||||
await files_lib.refresh_files()
|
||||
|
||||
archive1: Archive = await reset_and_get_archive()
|
||||
# No title/URL could be found.
|
||||
archive1: Archive = test_session.query(Archive).one()
|
||||
assert archive1.url is None
|
||||
assert archive1.file_group.url is None
|
||||
assert archive1.file_group.title is None
|
||||
|
||||
|
||||
|
|
@ -286,7 +282,7 @@ async def test_new_archive(test_session, fake_now):
|
|||
archive1 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
# Everything is filled out.
|
||||
assert isinstance(archive1, Archive)
|
||||
assert archive1.archive_datetime is not None
|
||||
assert archive1.file_group.download_datetime is not None
|
||||
assert isinstance(archive1.singlefile_path, pathlib.Path)
|
||||
assert isinstance(archive1.readability_path, pathlib.Path)
|
||||
assert isinstance(archive1.readability_txt_path, pathlib.Path)
|
||||
|
|
@ -520,7 +516,7 @@ async def test_refresh_archives(test_session, test_directory, test_async_client,
|
|||
assert response.json['file_groups'][0]['data'].get('readability_txt_path') == \
|
||||
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt', \
|
||||
'Could not find readability text file containing "text"'
|
||||
assert response.json['file_groups'][0]['data']['archive_datetime'], 'Archive has no datetime'
|
||||
assert response.json['file_groups'][0]['download_datetime'], 'Archive has no datetime'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -611,7 +607,7 @@ def test_archive_files(test_directory):
|
|||
def test_archive_order(test_session, test_directory, archive_factory):
|
||||
archive1 = archive_factory()
|
||||
archive2 = archive_factory()
|
||||
archive2.archive_datetime += timedelta(seconds=10)
|
||||
archive2.file_group.download_datetime += timedelta(seconds=10)
|
||||
test_session.commit()
|
||||
|
||||
assert archive2 > archive1
|
||||
|
|
|
|||
|
|
@ -30,9 +30,9 @@ def test_get_video(test_client, test_session, simple_channel, video_factory):
|
|||
"""Test that you get can information about a video. Test that video file can be gotten."""
|
||||
now_ = now()
|
||||
video1 = video_factory(channel_id=simple_channel.id, title='vid1')
|
||||
video1.upload_date = now_
|
||||
video1.file_group.published_datetime = now_
|
||||
video2 = video_factory(channel_id=simple_channel.id, title='vid2')
|
||||
video2.upload_date = now_ + timedelta(seconds=1)
|
||||
video2.file_group.published_datetime = now_ + timedelta(seconds=1)
|
||||
test_session.commit()
|
||||
|
||||
# Test that a 404 is returned when no video exists
|
||||
|
|
@ -276,7 +276,7 @@ async def test_channel_download_requires_refresh(test_session, download_channel,
|
|||
|
||||
Videos already downloaded are not downloaded again."""
|
||||
vid = video_factory(channel_id=download_channel.id, with_video_file=True, with_poster_ext='jpg')
|
||||
vid.url = 'https://example.com/1'
|
||||
vid.file_group.url = 'https://example.com/1'
|
||||
d = download_channel.get_download()
|
||||
d.next_download = None
|
||||
test_session.commit()
|
||||
|
|
@ -502,7 +502,7 @@ def test_search_videos_channel(test_client, test_session, video_factory):
|
|||
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
|
||||
|
||||
|
||||
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search):
|
||||
def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
|
||||
for i in range(50):
|
||||
video_factory(channel_id=simple_channel.id)
|
||||
|
||||
|
|
@ -524,7 +524,8 @@ def test_get_channel_videos_pagination(test_client, test_session, simple_channel
|
|||
]
|
||||
last_ids = []
|
||||
for offset, video_count in tests:
|
||||
_, response = assert_video_search(channel_id=simple_channel.id, order_by='upload_date', offset=offset, limit=20)
|
||||
_, response = assert_video_search(channel_id=simple_channel.id, order_by='published_datetime', offset=offset,
|
||||
limit=20)
|
||||
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
|
||||
current_ids = [i['id'] for i in response.json['file_groups']]
|
||||
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
||||
|
|
|
|||
|
|
@ -132,7 +132,7 @@ def video_factory(test_session, test_directory):
|
|||
video = Video.from_paths(test_session, *paths)
|
||||
video.channel_id = channel_id
|
||||
video.source_id = source_id or title
|
||||
video.upload_date = upload_date
|
||||
video.file_group.published_datetime = upload_date
|
||||
video.validate()
|
||||
return video
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from wrolpi.db import optional_session
|
|||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import get_no_channel_directory, get_videos_directory, update_view_counts, ffmpeg_video_complete
|
||||
|
|
@ -197,8 +198,8 @@ class VideoDownloader(Downloader, ABC):
|
|||
@optional_session
|
||||
def already_downloaded(self, *urls: str, session: Session = None) -> List:
|
||||
# We only consider a video record with a video file as "downloaded".
|
||||
videos = list(session.query(Video).filter(Video.url.in_(urls)))
|
||||
return videos
|
||||
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video'))
|
||||
return file_groups
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
if download.attempts >= 10:
|
||||
|
|
|
|||
|
|
@ -77,14 +77,14 @@ def validate_video(video: Video, channel_generate_poster: bool):
|
|||
found, it will be generated from the video file.
|
||||
"""
|
||||
info_json_path = video.info_json_path
|
||||
json_data_missing = bool(video.file_group.title) and bool(video.duration) and bool(video.view_count) \
|
||||
and bool(video.url)
|
||||
json_data_missing = bool(video.file_group.title) and bool(video.file_group.length) and bool(video.view_count) \
|
||||
and bool(video.file_group.url)
|
||||
if info_json_path and json_data_missing is False:
|
||||
# These properties can be found in the info json.
|
||||
video_info_json = process_video_info_json(video)
|
||||
video.file_group.title = video_info_json.title
|
||||
video.duration = video_info_json.duration
|
||||
video.url = video_info_json.url
|
||||
video.file_group.length = video_info_json.duration
|
||||
video.file_group.url = video_info_json.url
|
||||
# View count will probably be overwritten by more recent data when this Video's Channel is
|
||||
# updated.
|
||||
video.view_count = video.view_count or video_info_json.view_count
|
||||
|
|
@ -96,17 +96,17 @@ def validate_video(video: Video, channel_generate_poster: bool):
|
|||
if not video_path:
|
||||
logger.error(f'{video} does not have video_path!')
|
||||
|
||||
if video_path and (not video.file_group.title or not video.upload_date or not video.source_id):
|
||||
if video_path and (not video.file_group.title or not video.file_group.published_datetime or not video.source_id):
|
||||
# Video is missing things that can be extracted from the video file name.
|
||||
# These are the least trusted, so anything already on the video should be trusted.
|
||||
_, upload_date, source_id, title = parse_video_file_name(video_path)
|
||||
upload_date = upload_date or video.upload_date
|
||||
if upload_date and isinstance(upload_date, str):
|
||||
upload_date = dates.strpdate(upload_date)
|
||||
if not upload_date.tzinfo:
|
||||
upload_date = upload_date.astimezone(pytz.UTC)
|
||||
_, published_date, source_id, title = parse_video_file_name(video_path)
|
||||
published_date = published_date or video.file_group.published_datetime
|
||||
if published_date and isinstance(published_date, str):
|
||||
published_date = dates.strpdate(published_date)
|
||||
if not published_date.tzinfo:
|
||||
published_date = published_date.astimezone(pytz.UTC)
|
||||
video.file_group.title = video.file_group.title or html.unescape(title)
|
||||
video.upload_date = upload_date
|
||||
video.file_group.published_datetime = published_date
|
||||
video.source_id = video.source_id or source_id
|
||||
|
||||
if channel_generate_poster:
|
||||
|
|
@ -116,18 +116,18 @@ def validate_video(video: Video, channel_generate_poster: bool):
|
|||
# Poster was created/updated.
|
||||
video.file_group.append_files(new_poster_path)
|
||||
if duration:
|
||||
video.duration = duration
|
||||
video.file_group.length = duration
|
||||
|
||||
if video_path and not video.duration:
|
||||
if video_path and not video.file_group.length:
|
||||
# Duration was not retrieved during poster generation.
|
||||
if video.ffprobe_json:
|
||||
if duration := video.ffprobe_json['format'].get('duration'):
|
||||
video.duration = float(duration)
|
||||
video.file_group.length = float(duration)
|
||||
elif (video_streams := video.get_streams_by_codec_type('video')) and 'duration' in video_streams[0]:
|
||||
video.duration = float(video_streams[0]['duration'])
|
||||
video.file_group.length = float(video_streams[0]['duration'])
|
||||
else:
|
||||
# Slowest method.
|
||||
video.duration = get_video_duration(video_path)
|
||||
video.file_group.length = get_video_duration(video_path)
|
||||
|
||||
if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES:
|
||||
# Caption file was deleted, clear out old captions.
|
||||
|
|
@ -482,11 +482,11 @@ async def get_statistics():
|
|||
-- total videos
|
||||
COUNT(v.id) AS "videos",
|
||||
-- total videos downloaded over the past week/month/year
|
||||
COUNT(v.id) FILTER (WHERE upload_date >= current_date - interval '1 week') AS "week",
|
||||
COUNT(v.id) FILTER (WHERE upload_date >= current_date - interval '1 month') AS "month",
|
||||
COUNT(v.id) FILTER (WHERE upload_date >= current_date - interval '1 year') AS "year",
|
||||
COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 week') AS "week",
|
||||
COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 month') AS "month",
|
||||
COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 year') AS "year",
|
||||
-- sum of all video lengths in seconds
|
||||
COALESCE(SUM(v.duration), 0) AS "sum_duration",
|
||||
COALESCE(SUM(fg.length), 0) AS "sum_duration",
|
||||
-- sum of all video file sizes
|
||||
COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size",
|
||||
-- largest video
|
||||
|
|
@ -511,9 +511,9 @@ async def get_statistics():
|
|||
video
|
||||
LEFT JOIN file_group fg on video.file_group_id = fg.id
|
||||
WHERE
|
||||
video.upload_date >= date_trunc('month', months.a)
|
||||
AND video.upload_date < date_trunc('month', months.a) + interval '1 month'
|
||||
AND video.upload_date IS NOT NULL
|
||||
published_datetime >= date_trunc('month', months.a)
|
||||
AND published_datetime < date_trunc('month', months.a) + interval '1 month'
|
||||
AND published_datetime IS NOT NULL
|
||||
GROUP BY
|
||||
1
|
||||
ORDER BY
|
||||
|
|
@ -551,7 +551,7 @@ def parse_video_file_name(video_path: pathlib.Path) -> \
|
|||
"""
|
||||
A Video's file name can have data in it, this attempts to extract what may be there.
|
||||
|
||||
Example: {channel_name}_{upload_date}_{source_id}_title{ext}
|
||||
Example: {channel_name}_{published_datetime}_{source_id}_title{ext}
|
||||
"""
|
||||
video_str = str(video_path)
|
||||
if match := NAME_PARSER.match(video_str):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import datetime
|
||||
import json
|
||||
import pathlib
|
||||
from pathlib import Path
|
||||
|
|
@ -11,7 +10,6 @@ from sqlalchemy.orm.collections import InstrumentedList
|
|||
from modules.videos.errors import UnknownVideo, UnknownChannel
|
||||
from wrolpi.captions import read_captions
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task
|
||||
from wrolpi.dates import now, TZDateTime
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
|
||||
|
|
@ -29,13 +27,9 @@ class Video(ModelHelper, Base):
|
|||
__tablename__ = 'video'
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
duration = Column(Integer)
|
||||
source_id = Column(String)
|
||||
upload_date: datetime.datetime = Column(TZDateTime)
|
||||
url = Column(String)
|
||||
view_count = Column(Integer)
|
||||
viewed = Column(TZDateTime)
|
||||
ffprobe_json = deferred(Column(JSON))
|
||||
source_id = Column(String) # The id from yt-dlp
|
||||
view_count = Column(Integer) # The view count from the ChannelDownloader (or from initial download)
|
||||
ffprobe_json = deferred(Column(JSON)) # Data that is fetched once from ffprobe (ffmpeg)
|
||||
|
||||
channel_id = Column(Integer, ForeignKey('channel.id'))
|
||||
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
|
||||
|
|
@ -75,7 +69,6 @@ class Video(ModelHelper, Base):
|
|||
codec_names=codec_names,
|
||||
codec_types=codec_types,
|
||||
description=self.file_group.c_text or self.get_video_description(),
|
||||
duration=self.duration,
|
||||
id=self.id,
|
||||
info_json_file=self.info_json_file,
|
||||
info_json_path=self.info_json_path,
|
||||
|
|
@ -83,8 +76,6 @@ class Video(ModelHelper, Base):
|
|||
poster_path=self.poster_path,
|
||||
source_id=self.source_id,
|
||||
stem=split_path_stem_and_suffix(self.video_path)[0],
|
||||
upload_date=self.upload_date.isoformat() if self.upload_date else None,
|
||||
url=self.url,
|
||||
video_path=self.video_path,
|
||||
view_count=self.view_count,
|
||||
)
|
||||
|
|
@ -102,14 +93,11 @@ class Video(ModelHelper, Base):
|
|||
|
||||
def add_to_skip_list(self):
|
||||
"""Add this video to the DownloadManager's skip list."""
|
||||
if self.url:
|
||||
download_manager.add_to_skip_list(self.url)
|
||||
if self.file_group.url:
|
||||
download_manager.add_to_skip_list(self.file_group.url)
|
||||
else:
|
||||
logger.warning(f'{self} cannot be added to skip list because it does not have a URL')
|
||||
|
||||
def set_viewed(self):
|
||||
self.viewed = now()
|
||||
|
||||
def get_info_json(self) -> Optional[Dict]:
|
||||
"""If this Video has an info_json file, return it's contents. Otherwise, return None."""
|
||||
info_json_path = self.info_json_path
|
||||
|
|
@ -156,16 +144,17 @@ class Video(ModelHelper, Base):
|
|||
session = Session.object_session(self)
|
||||
|
||||
with get_db_curs() as curs:
|
||||
if self.upload_date:
|
||||
if self.file_group.published_datetime:
|
||||
# Get videos next to this Video's upload date.
|
||||
stmt = '''
|
||||
WITH numbered_videos AS (
|
||||
SELECT id,
|
||||
ROW_NUMBER() OVER (ORDER BY upload_date ASC) AS row_number
|
||||
FROM video
|
||||
SELECT fg.id,
|
||||
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
|
||||
FROM file_group fg
|
||||
LEFT OUTER JOIN video v on fg.id = v.file_group_id
|
||||
WHERE
|
||||
channel_id = %(channel_id)s
|
||||
AND upload_date IS NOT NULL
|
||||
v.channel_id = %(channel_id)s
|
||||
AND fg.published_datetime IS NOT NULL
|
||||
)
|
||||
SELECT id
|
||||
FROM numbered_videos
|
||||
|
|
@ -174,7 +163,7 @@ class Video(ModelHelper, Base):
|
|||
FROM numbered_videos
|
||||
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
|
||||
WHERE
|
||||
id = %(video_id)s
|
||||
id = %(fg_id)s
|
||||
)
|
||||
'''
|
||||
else:
|
||||
|
|
@ -198,7 +187,7 @@ class Video(ModelHelper, Base):
|
|||
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
|
||||
)
|
||||
'''
|
||||
curs.execute(stmt, dict(channel_id=self.channel_id, video_id=self.id))
|
||||
curs.execute(stmt, dict(channel_id=self.channel_id, fg_id=self.file_group_id))
|
||||
|
||||
results = [i[0] for i in curs.fetchall()]
|
||||
|
||||
|
|
|
|||
|
|
@ -91,7 +91,6 @@ class Video:
|
|||
poster_path: str
|
||||
source_id: str
|
||||
title: str
|
||||
upload_date: date
|
||||
video_path: str
|
||||
name: str
|
||||
caption_path: str
|
||||
|
|
@ -108,7 +107,6 @@ class VideoWithChannel:
|
|||
poster_path: str
|
||||
source_id: str
|
||||
title: str
|
||||
upload_date: date
|
||||
video_path: str
|
||||
name: str
|
||||
caption_path: str
|
||||
|
|
@ -119,11 +117,6 @@ class VideoWithChannel:
|
|||
channel = ChannelModel
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelVideosResponse:
|
||||
videos: List[Video]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtendedVideo(Video):
|
||||
info_json: dict
|
||||
|
|
@ -131,7 +124,7 @@ class ExtendedVideo(Video):
|
|||
|
||||
@dataclass
|
||||
class VideoResponse:
|
||||
file = ExtendedVideo
|
||||
file_group = ExtendedVideo
|
||||
prev = ExtendedVideo
|
||||
next = ExtendedVideo
|
||||
|
||||
|
|
|
|||
|
|
@ -228,7 +228,7 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit
|
|||
assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
|
||||
|
||||
|
||||
def test_search_videos(test_client, test_session, video_factory, assert_video_search, simple_channel, tag_factory):
|
||||
def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
|
||||
"""Search the Video table. This does not need to use a join with the File table."""
|
||||
vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
|
||||
vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2',
|
||||
|
|
@ -248,11 +248,11 @@ def test_search_videos(test_client, test_session, video_factory, assert_video_se
|
|||
assert test_session.query(Video).count() == 3
|
||||
assert test_session.query(FileGroup).count() == 3
|
||||
|
||||
assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='upload_date')
|
||||
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-upload_date')
|
||||
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-upload_date', limit=2)
|
||||
assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-upload_date', limit=2, offset=1)
|
||||
assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-upload_date', channel_id=simple_channel.id)
|
||||
assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
|
||||
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
|
||||
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
|
||||
assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2, offset=1)
|
||||
assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime', channel_id=simple_channel.id)
|
||||
assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
|
||||
assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
|
||||
assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
|
||||
|
|
|
|||
|
|
@ -272,4 +272,4 @@ async def test_video_ffprobe_json(test_session, video_file):
|
|||
|
||||
video = test_session.query(Video).one()
|
||||
assert video.ffprobe_json
|
||||
assert video.duration
|
||||
assert video.file_group.length
|
||||
|
|
|
|||
|
|
@ -398,7 +398,7 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
|
|||
video = Video.from_paths(test_session, video_file)
|
||||
video.channel_id = channel.id
|
||||
video.source_id = 'video 1 id'
|
||||
video.url = 'https://www.youtube.com/watch?v=video_1_url'
|
||||
video.file_group.url = 'https://www.youtube.com/watch?v=video_1_url'
|
||||
test_session.commit()
|
||||
|
||||
mock_video_extract_info.return_value = example_playlist_json # Playlist info is fetched first.
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ def test_validate_video(test_directory, video_factory, image_bytes_factory, vide
|
|||
assert vid1.video_path.stem == vid1.poster_path.stem
|
||||
assert vid1.poster_path.suffix == '.jpg'
|
||||
# File name date is assumed to be local timezone.
|
||||
assert vid1.upload_date and vid1.upload_date.year == 2005
|
||||
assert vid1.file_group.published_datetime and vid1.file_group.published_datetime.year == 2005
|
||||
assert vid1.source_id == '1234567890'
|
||||
assert vid1.file_group.title == 'The Title'
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
|
|||
"""
|
||||
with get_db_session(commit=True) as session:
|
||||
video = Video.find_by_id(video_id, session=session)
|
||||
video.set_viewed()
|
||||
video.file_group.set_viewed()
|
||||
previous_video, next_video = video.get_surrounding_videos()
|
||||
|
||||
video = video.__json__()
|
||||
|
|
@ -29,34 +29,33 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
|
|||
|
||||
|
||||
VIDEO_ORDERS = {
|
||||
'upload_date': 'v.upload_date ASC, LOWER(fg.primary_path) ASC',
|
||||
'-upload_date': 'v.upload_date DESC NULLS LAST, LOWER(fg.primary_path) ASC',
|
||||
'published_datetime': 'fg.published_datetime ASC, LOWER(fg.primary_path) ASC',
|
||||
'-published_datetime': 'fg.published_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC',
|
||||
'download_datetime': 'fg.download_datetime ASC, fg.published_datetime ASC, LOWER(fg.primary_path) ASC',
|
||||
'-download_datetime': 'fg.download_datetime DESC NULLS LAST, fg.published_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC',
|
||||
'rank': '2 DESC, LOWER(fg.primary_path) DESC',
|
||||
'-rank': '2 ASC, LOWER(fg.primary_path) ASC',
|
||||
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
|
||||
'-size': 'fg.size DESC, LOWER(fg.primary_path) DESC',
|
||||
'duration': 'duration ASC, LOWER(fg.primary_path) ASC',
|
||||
'-duration': 'duration DESC, LOWER(fg.primary_path) DESC',
|
||||
'viewed': 'v.viewed ASC',
|
||||
'-viewed': 'v.viewed DESC',
|
||||
'length': 'fg.length ASC, LOWER(fg.primary_path) ASC',
|
||||
'-length': 'fg.length DESC, LOWER(fg.primary_path) DESC',
|
||||
'viewed': 'fg.viewed ASC',
|
||||
'-viewed': 'fg.viewed DESC',
|
||||
'view_count': 'v.view_count ASC',
|
||||
'-view_count': 'v.view_count DESC',
|
||||
'modification_datetime': 'fg.modification_datetime ASC',
|
||||
'-modification_datetime': 'fg.modification_datetime DESC',
|
||||
}
|
||||
NO_NULL_ORDERS = {
|
||||
'viewed': 'v.viewed IS NOT NULL',
|
||||
'-viewed': 'v.viewed IS NOT NULL',
|
||||
'duration': 'v.duration IS NOT NULL',
|
||||
'-duration': 'v.duration IS NOT NULL',
|
||||
'viewed': 'fg.viewed IS NOT NULL',
|
||||
'-viewed': 'fg.viewed IS NOT NULL',
|
||||
'length': 'fg.length IS NOT NULL',
|
||||
'-length': 'fg.length IS NOT NULL',
|
||||
'size': 'fg.size IS NOT NULL',
|
||||
'-size': 'fg.size IS NOT NULL',
|
||||
'view_count': 'v.view_count IS NOT NULL',
|
||||
'-view_count': 'v.view_count IS NOT NULL',
|
||||
'modification_datetime': 'fg.modification_datetime IS NOT NULL',
|
||||
'-modification_datetime': 'fg.modification_datetime IS NOT NULL',
|
||||
}
|
||||
JOIN_ORDERS = ('upload_date', '-upload_date', 'viewed', '-viewed', 'view_count', '-view_count', 'duration', '-duration')
|
||||
JOIN_ORDERS = ('published_datetime', '-published_datetime', 'viewed', '-viewed', 'view_count', '-view_count',
|
||||
'length', '-length')
|
||||
DEFAULT_VIDEO_ORDER = 'rank'
|
||||
VIDEO_QUERY_LIMIT = 24
|
||||
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
|
|||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Video was added to skip list.
|
||||
assert test_download_manager.is_skipped(vid1.url)
|
||||
assert test_download_manager.is_skipped(vid1.file_group.url)
|
||||
# Video was deleted.
|
||||
assert test_session.query(Video).count() == 1
|
||||
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
|
||||
|
|
@ -138,7 +138,7 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
|
|||
request, response = await test_async_client.delete(f'/api/videos/video/{vid2.id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
assert test_download_manager.is_skipped(vid1.url, vid2.url)
|
||||
assert test_download_manager.is_skipped(vid1.file_group.url, vid2.file_group.url)
|
||||
assert test_session.query(Video).count() == 0
|
||||
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
|
||||
assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ async def rank_video_quality(video: Video) -> int:
|
|||
rank = 0
|
||||
|
||||
# These are more valuable.
|
||||
if video.url:
|
||||
if video.file_group.url:
|
||||
rank += 2
|
||||
if video.source_id:
|
||||
rank += 2
|
||||
|
|
|
|||
|
|
@ -106,12 +106,12 @@ async def create_download(url: str, downloader: str, message: str):
|
|||
|
||||
|
||||
async def download_or_ask_delete(video: Video, session: Session, message: str):
|
||||
if video.url and can_be_downloaded(video.url):
|
||||
await create_download(video.url, VideoDownloader.name, message)
|
||||
if video.file_group.url and can_be_downloaded(video.file_group.url):
|
||||
await create_download(video.file_group.url, VideoDownloader.name, message)
|
||||
# Always re-download when possible.
|
||||
do_delete = True
|
||||
else:
|
||||
logger.error(f'Video cannot be downloaded: {video.url or video}')
|
||||
logger.error(f'Video cannot be downloaded: {video.file_group.url or video}')
|
||||
do_delete = confirm(f'{message} Delete? (y/N)')
|
||||
|
||||
if do_delete:
|
||||
|
|
@ -140,7 +140,7 @@ async def find_corrupt_videos(channel_id: int = None):
|
|||
if not video.video_path.is_file():
|
||||
await download_or_ask_delete(video, session, f'Video does not exist: {video.video_path}')
|
||||
continue
|
||||
if video.url:
|
||||
if video.file_group.url:
|
||||
for path in video.file_group.my_paths():
|
||||
if not path.is_file() and path.name.endswith('.srt'):
|
||||
files = [i for i in video.file_group.my_files() if i['path'] != path]
|
||||
|
|
@ -148,7 +148,7 @@ async def find_corrupt_videos(channel_id: int = None):
|
|||
session.commit()
|
||||
logger.info(f'Removed non-existent srt: {video.video_path}')
|
||||
if not all(i.is_file() for i in video.file_group.my_paths()):
|
||||
await create_download(video.url, VideoDownloader.name,
|
||||
await create_download(video.file_group.url, VideoDownloader.name,
|
||||
f'Video is missing {path}: {video.video_path}')
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from enum import Enum
|
||||
from typing import Union
|
||||
|
|
|
|||
|
|
@ -45,18 +45,18 @@ EBOOK_MIMETYPES = (
|
|||
@dataclasses.dataclass
|
||||
class EBookData:
|
||||
cover: bytes = None
|
||||
creator: str = None
|
||||
author: str = None
|
||||
text: str = None
|
||||
title: str = None
|
||||
ebook_path: pathlib.Path = None
|
||||
|
||||
def __bool__(self):
|
||||
return bool(self.cover) or bool(self.creator) or bool(self.text) or bool(self.title)
|
||||
return bool(self.cover) or bool(self.author) or bool(self.text) or bool(self.title)
|
||||
|
||||
def __json__(self):
|
||||
d = {}
|
||||
if self.creator:
|
||||
d['creator'] = self.creator
|
||||
if self.author:
|
||||
d['author'] = self.author
|
||||
if self.title:
|
||||
d['title'] = self.title
|
||||
if self.ebook_path:
|
||||
|
|
@ -88,7 +88,7 @@ def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]
|
|||
if 'title' in value:
|
||||
data.title = data.title or value['title'][0][0]
|
||||
if 'creator' in value:
|
||||
data.creator = data.creator or value['creator'][0][0]
|
||||
data.author = data.author or value['creator'][0][0]
|
||||
|
||||
data.text = ''
|
||||
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||||
|
|
@ -150,8 +150,6 @@ class EBook(ModelHelper, Base):
|
|||
id = Column(Integer, primary_key=True)
|
||||
size: int = Column(Integer)
|
||||
|
||||
creator: str = Column(String)
|
||||
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
|
|
@ -239,8 +237,8 @@ def _model_ebook(ebook: EBook) -> EBook:
|
|||
if data:
|
||||
# Title is a_text.
|
||||
ebook.file_group.title = ebook.file_group.a_text = data.title
|
||||
# Creator is b_text.
|
||||
ebook.file_group.b_text = ebook.creator = data.creator
|
||||
# Author is b_text.
|
||||
ebook.file_group.b_text = ebook.file_group.author = data.author
|
||||
# All text is d_text.
|
||||
ebook.file_group.d_text = data.text
|
||||
d: dict = data.__json__()
|
||||
|
|
|
|||
|
|
@ -65,12 +65,13 @@ class FileGroup(ModelHelper, Base):
|
|||
__tablename__ = 'file_group'
|
||||
id: int = Column(BigInteger, primary_key=True)
|
||||
|
||||
# the Path of the file that can be modeled or indexed.
|
||||
author = Column(String) # name of the author, maybe even a URL
|
||||
data = Column(FancyJSON) # populated by the modeler
|
||||
download_datetime = Column(TZDateTime) # the date WROLPi downloaded this file.
|
||||
files = Column(FancyJSON, nullable=False) # populated during discovery
|
||||
idempotency = Column(TZDateTime) # used to track which files need to be deleted during refresh
|
||||
indexed = Column(Boolean, default=lambda: False, nullable=False) # wrolpi.files.lib.apply_indexers
|
||||
length = Column(BigInteger) # video duration, article words, etc.
|
||||
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
|
||||
model = Column(String) # "video", "archive", "ebook", etc.
|
||||
modification_datetime = Column(TZDateTime) # the modification date of the file on disk
|
||||
|
|
@ -79,6 +80,8 @@ class FileGroup(ModelHelper, Base):
|
|||
published_modified_datetime = Column(TZDateTime) # the date the publisher modified this file
|
||||
size = Column(BigInteger, default=lambda: 0)
|
||||
title = Column(String) # user-displayable title
|
||||
url = Column(String) # the location where this file can be downloaded.
|
||||
viewed = Column(TZDateTime) # the most recent time a User viewed this file.
|
||||
|
||||
tag_files: InstrumentedList = relationship('TagFile', cascade='all')
|
||||
|
||||
|
|
@ -103,23 +106,26 @@ class FileGroup(ModelHelper, Base):
|
|||
_, suffix = split_path_stem_and_suffix(self.primary_path)
|
||||
tags = sorted([i.tag.name for i in self.tag_files])
|
||||
d = {
|
||||
'author': self.author,
|
||||
'data': self.data,
|
||||
'directory': self.primary_path.parent,
|
||||
'download_datetime': self.download_datetime,
|
||||
'files': self.my_files(),
|
||||
'id': self.id,
|
||||
'key': self.primary_path,
|
||||
'length': self.length,
|
||||
'mimetype': self.mimetype,
|
||||
'model': self.model,
|
||||
'modified': self.modification_datetime or None,
|
||||
'name': self.primary_path.name,
|
||||
'primary_path': self.primary_path,
|
||||
'published_datetime': self.published_datetime,
|
||||
'published_modified_datetime': self.published_modified_datetime,
|
||||
'size': self.size,
|
||||
'suffix': suffix,
|
||||
'tags': tags,
|
||||
'title': self.title,
|
||||
'key': self.primary_path,
|
||||
'author': self.author,
|
||||
'published_datetime': self.published_datetime,
|
||||
'published_modified_datetime': self.published_modified_datetime,
|
||||
'url': self.url
|
||||
}
|
||||
return d
|
||||
|
||||
|
|
@ -129,6 +135,9 @@ class FileGroup(ModelHelper, Base):
|
|||
return self.title
|
||||
return self.primary_path.name
|
||||
|
||||
def set_viewed(self):
|
||||
self.viewed = now()
|
||||
|
||||
@optional_session
|
||||
def add_tag(self, tag: Tag, session: Session = None) -> TagFile:
|
||||
return tag.add_file_group_tag(self, session)
|
||||
|
|
|
|||
|
|
@ -21,15 +21,15 @@ async def test_index(test_session, test_directory, example_epub, example_mobi):
|
|||
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
assert ebook.file_group.mimetype.startswith(EPUB_MIMETYPE)
|
||||
assert ebook.creator == 'roland'
|
||||
assert ebook.file_group.author == 'roland'
|
||||
assert ebook.size == 292579
|
||||
assert ebook.file_group.data == {'creator': 'roland', 'title': 'WROLPi Test Book',
|
||||
assert ebook.file_group.data == {'author': 'roland', 'title': 'WROLPi Test Book',
|
||||
'cover_path': test_directory / 'ebook example.jpeg',
|
||||
'ebook_path': test_directory / 'ebook example.epub',
|
||||
}
|
||||
|
||||
assert ebook.file_group.a_text, 'Book title was not indexed'
|
||||
assert ebook.file_group.b_text, 'Book creator was not indexed'
|
||||
assert ebook.file_group.b_text, 'Book author was not indexed'
|
||||
assert ebook.file_group.d_text, 'Book text was not indexed'
|
||||
# Both ebooks are assumed to be the same book, but different formats.
|
||||
assert (epubs := ebook.file_group.my_files('application/epub+zip')) and len(epubs) == 1 \
|
||||
|
|
@ -86,7 +86,7 @@ def test_search(test_session, test_client, example_epub):
|
|||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.file_group.title == 'WROLPi Test Book'
|
||||
assert ebook.creator == 'roland'
|
||||
assert ebook.file_group.author == 'roland'
|
||||
assert ebook.file_group.a_text == 'WROLPi Test Book', 'Book title was not updated'
|
||||
assert ebook.file_group.d_text, 'Book was not indexed'
|
||||
|
||||
|
|
@ -100,7 +100,7 @@ def test_search(test_session, test_client, example_epub):
|
|||
assert_dict_contains(
|
||||
file_group['data'],
|
||||
{'cover_path': 'ebook example.jpeg', 'ebook_path': 'ebook example.epub', 'title': 'WROLPi Test Book',
|
||||
'creator': 'roland'},
|
||||
'author': 'roland'},
|
||||
)
|
||||
|
||||
# No Mobi ebook.
|
||||
|
|
@ -143,7 +143,7 @@ async def test_move_ebook(test_session, test_directory, example_epub, image_file
|
|||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.cover_path
|
||||
assert_dict_contains(ebook.file_group.data, dict(creator='roland'))
|
||||
assert_dict_contains(ebook.file_group.data, dict(author='roland'))
|
||||
assert sorted([i['path'] for i in ebook.file_group.files]) == [
|
||||
test_directory / 'ebook example.epub',
|
||||
test_directory / 'ebook example.jpg',
|
||||
|
|
@ -158,7 +158,7 @@ async def test_move_ebook(test_session, test_directory, example_epub, image_file
|
|||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.file_group.primary_path == test_directory / 'new/ebook example.epub'
|
||||
assert ebook.cover_path
|
||||
assert_dict_contains(ebook.file_group.data, dict(creator='roland'))
|
||||
assert_dict_contains(ebook.file_group.data, dict(author='roland'))
|
||||
assert sorted([i['path'] for i in ebook.file_group.files]) == [
|
||||
test_directory / 'new/ebook example.epub',
|
||||
test_directory / 'new/ebook example.jpg',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue