Moving data from models to FileGroup.

* Ebook.creator -> FileGroup.author
* Video.viewed -> FileGroup.viewed
* Video.duration -> FileGroup.length
* Video.url -> FileGroup.url
* Archive.url -> FileGroup.url
This commit is contained in:
Roland 2023-10-24 10:20:06 -06:00
parent ae9d188391
commit 15687e534a
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
32 changed files with 418 additions and 209 deletions

View file

@ -0,0 +1,49 @@
"""eBook creator.
Revision ID: 365c985893b8
Revises: 5ad9efbf3570
Create Date: 2023-10-29 16:54:28.319427
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '365c985893b8'
down_revision = '5ad9efbf3570'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('''
UPDATE file_group
SET author=creator
FROM ebook
WHERE ebook.file_group_id = file_group.id
''')
session.execute('DROP INDEX IF EXISTS ebook_creator_idx')
session.execute('ALTER TABLE ebook DROP COLUMN IF EXISTS creator')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE ebook ADD COLUMN IF NOT EXISTS creator TEXT')
session.execute('''
UPDATE ebook
SET creator=author
FROM file_group
WHERE
ebook.file_group_id = file_group.id
AND model = 'ebook'
''')
session.execute('CREATE INDEX IF NOT EXISTS ebook_creator_idx ON ebook(creator)')

View file

@ -0,0 +1,141 @@
"""
Migrates Video.duration/.url/.viewed/Archive.url to FileGroup.length/.url/.viewed.
Revision ID: 5ad9efbf3570
Revises: 4b54ad052e2e
Create Date: 2023-10-24 09:25:33.818373
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '5ad9efbf3570'
down_revision = '4b54ad052e2e'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE file_group ADD COLUMN length INT DEFAULT 0')
session.execute('''
UPDATE file_group
SET length=video.duration
FROM video
WHERE file_group.id = video.file_group_id
''')
# Migrate Archive.url and Video.url to FileGroup.url.
session.execute('ALTER TABLE file_group ADD COLUMN url TEXT')
session.execute('''
UPDATE file_group
SET url=archive.url
FROM archive
WHERE file_group.id = archive.file_group_id
''')
session.execute('''
UPDATE file_group
SET url=video.url
FROM video
WHERE file_group.id = video.file_group_id
''')
# Migrate Archive.archive_datetime to FileGroup.download_datetime.
session.execute('ALTER TABLE file_group ADD COLUMN download_datetime TIMESTAMP WITH TIME ZONE')
session.execute('''
UPDATE file_group
SET download_datetime=archive.archive_datetime
FROM archive
WHERE file_group.id = archive.file_group_id
''')
# Migrate Video.viewed to FileGroup.viewed.
session.execute('ALTER TABLE file_group ADD COLUMN viewed TIMESTAMP WITH TIME ZONE')
session.execute('''
UPDATE file_group
SET viewed=video.viewed
FROM video
WHERE file_group.id = video.file_group_id
''')
session.execute('DROP INDEX IF EXISTS archive_archive_datetime_idx')
session.execute('DROP INDEX IF EXISTS archive_url_idx')
session.execute('DROP INDEX IF EXISTS video_url_idx')
session.execute('DROP INDEX IF EXISTS video_viewed_idx')
session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS archive_datetime')
session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS url')
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS url')
session.execute('ALTER TABLE video DROP COLUMN IF EXISTS viewed')
session.execute('CREATE INDEX file_group_download_datetime_idx ON file_group(download_datetime)')
session.execute('CREATE INDEX file_group_length_idx ON file_group(length)')
session.execute('CREATE INDEX file_group_url_idx ON file_group(url)')
session.execute('CREATE INDEX file_group_viewed_idx ON file_group(viewed)')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS duration INTEGER')
session.execute('''
UPDATE video
SET duration=file_group.length
FROM file_group
WHERE file_group.id = video.file_group_id
''')
session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS url TEXT')
session.execute('''
UPDATE video
SET url=file_group.url
FROM file_group
WHERE file_group.id = video.file_group_id
''')
# Video has `viewed`, Archive does not.
session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS viewed TIMESTAMP WITH TIME ZONE')
session.execute('''
UPDATE video
SET url=file_group.viewed
FROM file_group
WHERE file_group.id = video.file_group_id
''')
session.execute('ALTER TABLE archive ADD COLUMN IF NOT EXISTS url TEXT')
session.execute('''
UPDATE archive
SET url=file_group.url
FROM file_group
WHERE file_group.id = archive.file_group_id
''')
session.execute('ALTER TABLE archive ADD COLUMN IF NOT EXISTS archive_datetime TIMESTAMP WITH TIME ZONE')
session.execute('''
UPDATE archive
SET archive_datetime=file_group.download_datetime
FROM file_group
WHERE file_group.id = archive.file_group_id
''')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS download_datetime')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS length')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS url')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS viewed')
session.execute('DROP INDEX IF EXISTS file_group_download_datetime_idx')
session.execute('DROP INDEX IF EXISTS file_group_length_idx')
session.execute('DROP INDEX IF EXISTS file_group_url_idx')
session.execute('DROP INDEX IF EXISTS file_group_viewed_idx')
session.execute('CREATE INDEX IF NOT EXISTS archive_archive_datetime_idx ON archive(archive_datetime)')
session.execute('CREATE INDEX IF NOT EXISTS archive_url_idx ON archive(url)')
session.execute('CREATE INDEX IF NOT EXISTS video_duration_idx ON video(duration)')
session.execute('CREATE INDEX IF NOT EXISTS video_url_idx ON video(url)')
session.execute('CREATE INDEX IF NOT EXISTS video_viewed_idx ON video(viewed)')

View file

@ -25,6 +25,15 @@ def upgrade():
session.execute('ALTER TABLE file_group ADD COLUMN author TEXT')
session.execute('ALTER TABLE file_group ADD COLUMN published_datetime TIMESTAMP WITH TIME ZONE')
session.execute('ALTER TABLE file_group ADD COLUMN published_modified_datetime TIMESTAMP WITH TIME ZONE')
# Migrate Video.upload_date to FileGroup.published_datetime.
session.execute('''
UPDATE file_group
SET published_datetime=upload_date
FROM video
WHERE file_group.id = video.file_group_id
''')
session.execute('CREATE INDEX IF NOT EXISTS file_group_author_idx ON file_group(author)')
session.execute('CREATE INDEX IF NOT EXISTS file_group_published_datetime_idx ON file_group(published_datetime)')
session.execute('CREATE INDEX IF NOT EXISTS file_group_published_modified_datetime_idx'
@ -34,9 +43,19 @@ def upgrade():
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
# Migrate FileGroup.published_datetime to Video.upload_date.
session.execute('''
UPDATE video
SET upload_date=published_datetime
FROM file_group
WHERE file_group.id = video.file_group_id
''')
session.execute('DROP INDEX IF EXISTS file_group_author_idx')
session.execute('DROP INDEX IF EXISTS file_group_published_datetime_idx')
session.execute('DROP INDEX IF EXISTS file_group_published_modified_datetime_idx')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS author')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_datetime')
session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_modified_datetime')

View file

@ -69,10 +69,7 @@ function ArchivePage() {
const {archiveFile, history, fetchArchive} = useArchive(archiveId);
const {theme} = useContext(ThemeContext);
let title;
if (archiveFile && archiveFile.archive) {
title = archiveFile.archive.title;
}
let title = archiveFile ? archiveFile.title ? archiveFile.title : archiveFile.name : null;
useTitle(title);
if (archiveFile === null) {
@ -168,7 +165,7 @@ function ArchivePage() {
await fetchArchive();
}
const archivedDatetimeString = isoDatetimeToString(data.archive_datetime, true);
const archivedDatetimeString = isoDatetimeToString(archiveFile.download_datetime, true);
const publishedDatetimeString = archiveFile.published_datetime ? isoDatetimeToString(archiveFile.published_datetime, true)
: 'unknown';
const modifiedDatetimeString = archiveFile.published_modified_datetime
@ -305,7 +302,7 @@ export function ArchiveCard({file}) {
</CardLink>}
<CardMeta {...s}>
<p>
{isoDatetimeToString(data.archive_datetime)}
{isoDatetimeToString(file.published_datetime)}
</p>
</CardMeta>
<CardDescription>
@ -426,8 +423,11 @@ function ArchivesPage() {
} = useSearchArchives();
let archiveOrders = [
{value: 'date', text: 'Date'},
{value: 'published_datetime', text: 'Published Date', short: 'P.Date'},
{value: 'published_modified_datetime', text: 'Modified Date', short: 'M.Date'},
{value: 'download_datetime', text: 'Download Date', short: 'D.Date'},
{value: 'size', text: 'Size'},
{value: 'viewed', text: 'Recently Viewed', short: 'R.Viewed'},
];
if (searchStr) {

View file

@ -148,27 +148,26 @@ export function isoDatetimeToElapsedPopup(dt) {
return secondsToElapsedPopup(d.getTime() / 1000);
}
export function secondsToDuration(video) {
let duration = video.duration;
let hours = Math.floor(duration / 3600);
duration -= hours * 3600;
let minutes = Math.floor(duration / 60);
let seconds = duration - (minutes * 60);
export function secondsToHMS(totalSeconds) {
let hours = Math.floor(totalSeconds / 3600);
totalSeconds -= hours * 3600;
let minutes = Math.floor(totalSeconds / 60);
let seconds = totalSeconds - (minutes * 60);
hours = String('00' + hours).slice(-2);
minutes = String('00' + minutes).slice(-2);
seconds = String('00' + seconds).slice(-2);
return [hours, minutes, seconds];
return {hours, minutes, seconds};
}
export function Duration({video}) {
let [hours, minutes, seconds] = secondsToDuration(video);
export function Duration({totalSeconds}) {
const {hours, minutes, seconds} = secondsToHMS(totalSeconds);
if (hours > 0) {
return <div className="duration-overlay">{hours}:{minutes}:{seconds}</div>
} else if (video.duration) {
return <div className="duration-overlay">{minutes}:{seconds}</div>
return <div className='duration-overlay'>{hours}:{minutes}:{seconds}</div>
} else if (totalSeconds) {
return <div className='duration-overlay'>{minutes}:{seconds}</div>
} else {
return <></>
}
@ -224,7 +223,7 @@ export function RequiredAsterisk() {
return <span style={{color: '#db2828'}}> *</span>
}
export let defaultVideoOrder = '-upload_date';
export let defaultVideoOrder = '-published_datetime';
export let defaultSearchOrder = 'rank';
export const frequencyOptions = [{key: null, text: '', value: null}, {
@ -1277,7 +1276,7 @@ export function SortButton({sorts = []}) {
</Modal>
<ButtonGroup icon>
<Button icon={desc ? 'sort down' : 'sort up'} onClick={() => toggleDesc()}/>
<Button content={selectedSort['text']} onClick={() => setOpen(true)}/>
<Button content={selectedSort['short'] || selectedSort['text']} onClick={() => setOpen(true)}/>
</ButtonGroup>
</>
}

View file

@ -70,7 +70,6 @@ import {Headlines} from "./Headline";
function EbookCard({file}) {
const {s} = useContext(ThemeContext);
let {data} = file;
const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
const isEpub = file['mimetype'].startsWith('application/epub');
@ -83,12 +82,12 @@ function EbookCard({file}) {
<CardHeader>
<Container textAlign='left'>
<ExternalCardLink to={viewerUrl || downloadUrl}>
{cardTitleWrapper(data ? data.title : file.title)}
{cardTitleWrapper(file ? file.title : file.title)}
</ExternalCardLink>
</Container>
</CardHeader>
<CardMeta>
{data.creator ? <b {...s}>{data.creator}</b> : null}
{file.author ? <b {...s}>{file.author}</b> : null}
{file.size && <p {...s}>{humanFileSize(file.size)}</p>}
</CardMeta>
</CardContent>

View file

@ -73,7 +73,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
const navigate = useNavigate();
const videoRef = React.useRef();
useTitle(videoFile === null || videoFile === undefined ? null : videoFile.video.title);
useTitle(videoFile ? videoFile.title ? videoFile.title : videoFile.name : null);
let video;
if (videoFile) {
video = videoFile.video;
@ -241,7 +241,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
<Segment>
<Header as='h2'>{title}</Header>
{video.upload_date && <h3>{isoDatetimeToString(video.upload_date)}</h3>}
{videoFile.published_datetime && <h3>{isoDatetimeToString(videoFile.published_datetime)}</h3>}
<h3>
{channel && <Link to={`/videos/channel/${channel.id}/video`}>
{channel.name}

View file

@ -84,10 +84,11 @@ function VideosPage() {
useTitle(title);
let videoOrders = [
{value: 'upload_date', text: 'Date'},
{value: 'duration', text: 'Duration'},
{value: 'published_datetime', text: 'Published Date', short: 'P.Date'},
{value: 'length', text: 'Length'},
{value: 'size', text: 'Size'},
{value: 'view_count', text: 'Views'},
{value: 'viewed', text: 'Recently Viewed', short: 'R.Viewed'},
]
if (searchStr) {
@ -291,7 +292,7 @@ export function VideoCard({file}) {
// Default to video FilePreview for lone video files.
let video_url;
const upload_date = isoDatetimeToString(video.upload_date);
const published_datetime = isoDatetimeToString(file.published_datetime);
// A video may not have a channel.
const channel = video.channel ? video.channel : null;
let channel_url = null;
@ -320,7 +321,7 @@ export function VideoCard({file}) {
const color = mimetypeColor(file.mimetype);
return <Card color={color}>
{poster}
<Duration video={video}/>
<Duration totalSeconds={file.length}/>
<CardContent {...s}>
<CardHeader>
<Container textAlign='left'>
@ -332,7 +333,7 @@ export function VideoCard({file}) {
{channel && <Link to={channel_url} className="no-link-underscore card-link">
<b {...s}>{channel.name}</b>
</Link>}
<p {...s}>{upload_date}</p>
<p {...s}>{published_datetime}</p>
</Container>
</CardDescription>
</CardContent>

View file

@ -61,8 +61,8 @@ class ArchiveDownloader(Downloader, ABC):
@optional_session
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
archives = list(session.query(Archive).filter(Archive.url.in_(urls)))
return archives
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
return file_groups
async def do_singlefile(self, download: Download) -> bytes:
"""Create a Singlefile from the archive's URL."""
@ -226,14 +226,12 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
file_group.d_text = contents
file_group.data = {
'id': archive.id,
'archive_datetime': archive.archive_datetime,
'domain': archive.domain.domain if archive.domain else None,
'readability_json_path': archive.readability_json_path,
'readability_path': archive.readability_path,
'readability_txt_path': archive.readability_txt_path,
'screenshot_path': archive.screenshot_path,
'singlefile_path': archive.singlefile_path,
'url': archive.url,
}
return archive

View file

@ -40,8 +40,8 @@ def archive_factory(test_session, archive_directory, make_files_structure):
domain_dir = archive_directory / domain_dir
domain_dir.mkdir(exist_ok=True)
archive_datetime = next(now)
timestamp = archive_strftime(archive_datetime)
download_datetime = next(now)
timestamp = archive_strftime(download_datetime)
json_contents = {}
if title != 'NA':
@ -76,7 +76,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
archive = Archive.from_paths(test_session, *files)
archive.url = url
archive.title = title
archive.archive_datetime = archive.file_group.published_datetime = next(now)
archive.file_group.download_datetime = archive.file_group.published_datetime = next(now)
archive.file_group.modification_datetime = next(now)
archive.domain = domain
archive.validate()

View file

@ -198,7 +198,7 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
with get_db_session(commit=True) as session:
archive = Archive.from_paths(session, *paths)
archive.archive_datetime = now()
archive.file_group.download_datetime = now()
archive.url = url
archive.domain = get_or_create_domain(session, url)
session.flush()
@ -378,6 +378,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
def get_archive(session, archive_id: int) -> Archive:
"""Get an Archive."""
archive = Archive.find_by_id(archive_id, session=session)
archive.file_group.set_viewed()
return archive
@ -439,20 +440,38 @@ def get_domains():
ARCHIVE_ORDERS = {
'date': (date := 'fg.published_datetime ASC, a.archive_datetime ASC'),
'-date': (_date := 'fg.published_datetime DESC NULLS LAST, a.archive_datetime DESC NULLS LAST'),
'published_datetime': (date := 'fg.published_datetime ASC, fg.download_datetime ASC'),
'-published_datetime': (_date := 'fg.published_datetime DESC NULLS LAST, fg.download_datetime DESC NULLS LAST'),
'published_modified_datetime': f'fg.published_modified_datetime ASC NULLS LAST, {date}',
'-published_modified_datetime': f'fg.published_modified_datetime DESC, {_date}',
'download_datetime': 'fg.download_datetime ASC',
'-download_datetime': 'fg.download_datetime DESC',
'rank': f'2 DESC, {_date}',
'-rank': f'2 ASC, {date}',
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
'-size': 'fg.size DESC NULLS LAST, LOWER(fg.primary_path) DESC',
'viewed': 'fg.viewed ASC',
'-viewed': 'fg.viewed DESC',
}
ORDER_GROUP_BYS = {
'date': 'fg.published_datetime, a.archive_datetime',
'-date': 'fg.published_datetime, a.archive_datetime',
'rank': 'fg.published_datetime, a.archive_datetime',
'-rank': 'fg.published_datetime, a.archive_datetime',
'published_datetime': 'fg.published_datetime, fg.download_datetime',
'-published_datetime': 'fg.published_datetime, fg.download_datetime',
'published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime',
'-published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime',
'download_datetime': 'fg.download_datetime',
'-download_datetime': 'fg.download_datetime',
'rank': 'fg.published_datetime, fg.download_datetime',
'-rank': 'fg.published_datetime, fg.download_datetime',
'size': 'fg.size, fg.published_datetime, fg.primary_path',
'-size': 'fg.size, fg.published_datetime, fg.primary_path',
'viewed': 'fg.viewed',
'-viewed': 'fg.viewed',
}
NO_NULL_ORDERS = {
'viewed': 'fg.viewed IS NOT NULL',
'-viewed': 'fg.viewed IS NOT NULL',
'published_modified_datetime': 'fg.published_modified_datetime IS NOT NULL',
'-published_modified_datetime': 'fg.published_modified_datetime IS NOT NULL',
}
@ -461,10 +480,10 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
-> Tuple[List[dict], int]:
# Always filter FileGroups to Archives.
wheres = []
group_by = 'fg.published_datetime, a.archive_datetime, a.file_group_id'
group_by = 'fg.published_datetime, fg.download_datetime, a.file_group_id'
params = dict(search_str=search_str, offset=int(offset), limit=int(limit))
order_by = ARCHIVE_ORDERS['-date']
order_by = ARCHIVE_ORDERS['-published_datetime']
select_columns = ''
if search_str:
@ -480,7 +499,9 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
order_by = ARCHIVE_ORDERS[order]
group_by = f'{group_by}, {ORDER_GROUP_BYS[order]}'
except KeyError:
raise InvalidOrderBy(f'Invalid order by: {order}')
raise InvalidOrderBy(f'Invalid order byy {order}')
if order in NO_NULL_ORDERS:
wheres.append(NO_NULL_ORDERS[order])
if tag_names:
# Filter all FileGroups by those that have been tagged with the provided tag names.

View file

@ -10,7 +10,6 @@ from sqlalchemy.orm.collections import InstrumentedList
from wrolpi import dates
from wrolpi.common import ModelHelper, Base, logger
from wrolpi.dates import TZDateTime
from wrolpi.db import optional_session
from wrolpi.errors import UnknownArchive
from wrolpi.files.models import FileGroup
@ -31,9 +30,6 @@ class Archive(Base, ModelHelper):
__tablename__ = 'archive'
id = Column(Integer, primary_key=True)
archive_datetime = Column(TZDateTime)
url = Column(String)
domain_id = Column(Integer, ForeignKey('domains.id'))
domain = relationship('Domain', primaryjoin='Archive.domain_id==Domain.id')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
@ -41,20 +37,20 @@ class Archive(Base, ModelHelper):
def __repr__(self):
if self.domain:
return f'<Archive id={self.id} url={self.url} singlefile={repr(str(self.singlefile_path))} ' \
return f'<Archive id={self.id} url={self.file_group.url} singlefile={repr(str(self.singlefile_path))} ' \
f'domain={self.domain.domain}>'
return f'<Archive id={self.id} url={self.url} singlefile={repr(str(self.singlefile_path))}>'
return f'<Archive id={self.id} url={self.file_group.url} singlefile={repr(str(self.singlefile_path))}>'
@staticmethod
@optional_session
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
def get_by_id(id_: int, session: Session = None) -> Optional['Archive']:
"""Attempt to find an Archive with the provided id. Returns None if it cannot be found."""
archive = session.query(Archive).filter_by(id=id_).one_or_none()
return archive
@staticmethod
@optional_session
def find_by_id(id_: int, session: Session = None) -> Base:
def find_by_id(id_: int, session: Session = None) -> 'Archive':
"""Find an Archive with the provided id, raises an exception when no Archive is found.
@raise UnknownArchive: if the Archive can not be found"""
@ -84,7 +80,7 @@ class Archive(Base, ModelHelper):
if not isinstance(other, Archive):
raise ValueError(f'Cannot compare {type(other)} to Archive!')
return self.archive_datetime > other.archive_datetime
return self.file_group.published_datetime > other.file_group.published_datetime
def delete(self):
self.file_group.delete()
@ -105,9 +101,11 @@ class Archive(Base, ModelHelper):
session = Session.object_session(self)
history = list(session.query(Archive).filter(
Archive.id != self.id,
Archive.url == self.url,
Archive.url != None, # noqa
).order_by(Archive.archive_datetime))
FileGroup.url == self.file_group.url,
FileGroup.url != None, # noqa
) \
.join(FileGroup, FileGroup.id == Archive.file_group_id) \
.order_by(FileGroup.published_datetime))
return history
@property
@ -192,7 +190,7 @@ class Archive(Base, ModelHelper):
raise InvalidArchive() from e
# Readability is most trusted, it should overwrite any previous data.
self.url = url or self.url
self.file_group.url = url or self.file_group.url
self.file_group.title = title or self.file_group.title
def apply_singlefile_data(self):
@ -202,7 +200,7 @@ class Archive(Base, ModelHelper):
# Can't read contents of nothing.
raise ValueError('Cannot read singlefile data when this has no files')
if self.url and self.archive_datetime:
if self.file_group.url and self.file_group.published_datetime:
# This data has already been read.
return
@ -212,21 +210,21 @@ class Archive(Base, ModelHelper):
logger.error(f'Could not find SingleFile header in {self}')
return
if not self.url:
if not self.file_group.url:
try:
if match := MATCH_URL.findall(head):
self.url = match[0].strip()
self.file_group.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
if not self.archive_datetime:
if not self.file_group.download_datetime:
try:
if match := MATCH_DATE.findall(head):
dt = match[0].strip()
dt = ' '.join(dt.split(' ')[:5])
# SingleFile uses GMT.
dt = dates.strpdate(dt).replace(tzinfo=pytz.timezone('GMT'))
self.archive_datetime = dt
self.file_group.download_datetime = dt
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
@ -234,11 +232,11 @@ class Archive(Base, ModelHelper):
"""Get the domain from the URL."""
from modules.archive.lib import get_or_create_domain
domain = None
if self.url:
if self.file_group.url:
session = Session.object_session(self)
if not session:
raise ValueError('No session found!')
domain = get_or_create_domain(session, self.url)
domain = get_or_create_domain(session, self.file_group.url)
# Clear domain if the URL is missing.
self.domain_id = domain.id if domain else None

View file

@ -14,7 +14,6 @@ class ArchiveDict:
readability_txt_path: str
screenshot_path: str
title: str
archive_datetime: datetime
@dataclass

View file

@ -88,37 +88,33 @@ async def test_archive_title(test_session, archive_factory, singlefile_contents_
title='json title',
singlefile_contents=singlefile_contents_factory('singlefile title', 'https://example.com/singlefile-url'),
)
test_session.query(FileGroup).delete()
test_session.commit()
await files_lib.refresh_files()
async def reset_and_get_archive():
test_session.query(FileGroup).delete()
test_session.commit()
await files_lib.refresh_files()
return test_session.query(Archive).one()
archive1: Archive = await reset_and_get_archive()
assert test_session.query(FileGroup).count() == 1
assert test_session.query(Archive).count() == 1
archive1: Archive = test_session.query(Archive).one()
# Readability JSON is trusted first.
assert archive1.url == 'https://example.com/json-url'
assert archive1.file_group.url == 'https://example.com/json-url'
assert archive1.file_group.title == 'json title'
# Delete JSON file.
archive1.readability_json_path.unlink()
test_session.delete(archive1)
await files_lib.refresh_files()
archive1: Archive = test_session.query(Archive).one()
assert archive1.url == 'https://example.com/singlefile-url'
archive1: Archive = await reset_and_get_archive()
assert archive1.file_group.url == 'https://example.com/singlefile-url'
assert archive1.file_group.title == 'singlefile title'
# Clear out title and URL from Singlefile.
archive1.singlefile_path.write_text(singlefile_contents_factory('', ''))
test_session.delete(archive1)
await files_lib.refresh_files()
archive1: Archive = await reset_and_get_archive()
# No title/URL could be found.
archive1: Archive = test_session.query(Archive).one()
assert archive1.url is None
assert archive1.file_group.url is None
assert archive1.file_group.title is None
@ -286,7 +282,7 @@ async def test_new_archive(test_session, fake_now):
archive1 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
# Everything is filled out.
assert isinstance(archive1, Archive)
assert archive1.archive_datetime is not None
assert archive1.file_group.download_datetime is not None
assert isinstance(archive1.singlefile_path, pathlib.Path)
assert isinstance(archive1.readability_path, pathlib.Path)
assert isinstance(archive1.readability_txt_path, pathlib.Path)
@ -520,7 +516,7 @@ async def test_refresh_archives(test_session, test_directory, test_async_client,
assert response.json['file_groups'][0]['data'].get('readability_txt_path') == \
'archive/example.com/2021-10-05-16-20-10_NA.readability.txt', \
'Could not find readability text file containing "text"'
assert response.json['file_groups'][0]['data']['archive_datetime'], 'Archive has no datetime'
assert response.json['file_groups'][0]['download_datetime'], 'Archive has no datetime'
@pytest.mark.asyncio
@ -611,7 +607,7 @@ def test_archive_files(test_directory):
def test_archive_order(test_session, test_directory, archive_factory):
archive1 = archive_factory()
archive2 = archive_factory()
archive2.archive_datetime += timedelta(seconds=10)
archive2.file_group.download_datetime += timedelta(seconds=10)
test_session.commit()
assert archive2 > archive1

View file

@ -30,9 +30,9 @@ def test_get_video(test_client, test_session, simple_channel, video_factory):
"""Test that you get can information about a video. Test that video file can be gotten."""
now_ = now()
video1 = video_factory(channel_id=simple_channel.id, title='vid1')
video1.upload_date = now_
video1.file_group.published_datetime = now_
video2 = video_factory(channel_id=simple_channel.id, title='vid2')
video2.upload_date = now_ + timedelta(seconds=1)
video2.file_group.published_datetime = now_ + timedelta(seconds=1)
test_session.commit()
# Test that a 404 is returned when no video exists
@ -276,7 +276,7 @@ async def test_channel_download_requires_refresh(test_session, download_channel,
Videos already downloaded are not downloaded again."""
vid = video_factory(channel_id=download_channel.id, with_video_file=True, with_poster_ext='jpg')
vid.url = 'https://example.com/1'
vid.file_group.url = 'https://example.com/1'
d = download_channel.get_download()
d.next_download = None
test_session.commit()
@ -502,7 +502,7 @@ def test_search_videos_channel(test_client, test_session, video_factory):
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search):
def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
for i in range(50):
video_factory(channel_id=simple_channel.id)
@ -524,7 +524,8 @@ def test_get_channel_videos_pagination(test_client, test_session, simple_channel
]
last_ids = []
for offset, video_count in tests:
_, response = assert_video_search(channel_id=simple_channel.id, order_by='upload_date', offset=offset, limit=20)
_, response = assert_video_search(channel_id=simple_channel.id, order_by='published_datetime', offset=offset,
limit=20)
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
current_ids = [i['id'] for i in response.json['file_groups']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'

View file

@ -132,7 +132,7 @@ def video_factory(test_session, test_directory):
video = Video.from_paths(test_session, *paths)
video.channel_id = channel_id
video.source_id = source_id or title
video.upload_date = upload_date
video.file_group.published_datetime = upload_date
video.validate()
return video

View file

@ -20,6 +20,7 @@ from wrolpi.db import optional_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.lib import glob_shared_stem
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import get_no_channel_directory, get_videos_directory, update_view_counts, ffmpeg_video_complete
@ -197,8 +198,8 @@ class VideoDownloader(Downloader, ABC):
@optional_session
def already_downloaded(self, *urls: str, session: Session = None) -> List:
# We only consider a video record with a video file as "downloaded".
videos = list(session.query(Video).filter(Video.url.in_(urls)))
return videos
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video'))
return file_groups
async def do_download(self, download: Download) -> DownloadResult:
if download.attempts >= 10:

View file

@ -77,14 +77,14 @@ def validate_video(video: Video, channel_generate_poster: bool):
found, it will be generated from the video file.
"""
info_json_path = video.info_json_path
json_data_missing = bool(video.file_group.title) and bool(video.duration) and bool(video.view_count) \
and bool(video.url)
json_data_missing = bool(video.file_group.title) and bool(video.file_group.length) and bool(video.view_count) \
and bool(video.file_group.url)
if info_json_path and json_data_missing is False:
# These properties can be found in the info json.
video_info_json = process_video_info_json(video)
video.file_group.title = video_info_json.title
video.duration = video_info_json.duration
video.url = video_info_json.url
video.file_group.length = video_info_json.duration
video.file_group.url = video_info_json.url
# View count will probably be overwritten by more recent data when this Video's Channel is
# updated.
video.view_count = video.view_count or video_info_json.view_count
@ -96,17 +96,17 @@ def validate_video(video: Video, channel_generate_poster: bool):
if not video_path:
logger.error(f'{video} does not have video_path!')
if video_path and (not video.file_group.title or not video.upload_date or not video.source_id):
if video_path and (not video.file_group.title or not video.file_group.published_datetime or not video.source_id):
# Video is missing things that can be extracted from the video file name.
# These are the least trusted, so anything already on the video should be trusted.
_, upload_date, source_id, title = parse_video_file_name(video_path)
upload_date = upload_date or video.upload_date
if upload_date and isinstance(upload_date, str):
upload_date = dates.strpdate(upload_date)
if not upload_date.tzinfo:
upload_date = upload_date.astimezone(pytz.UTC)
_, published_date, source_id, title = parse_video_file_name(video_path)
published_date = published_date or video.file_group.published_datetime
if published_date and isinstance(published_date, str):
published_date = dates.strpdate(published_date)
if not published_date.tzinfo:
published_date = published_date.astimezone(pytz.UTC)
video.file_group.title = video.file_group.title or html.unescape(title)
video.upload_date = upload_date
video.file_group.published_datetime = published_date
video.source_id = video.source_id or source_id
if channel_generate_poster:
@ -116,18 +116,18 @@ def validate_video(video: Video, channel_generate_poster: bool):
# Poster was created/updated.
video.file_group.append_files(new_poster_path)
if duration:
video.duration = duration
video.file_group.length = duration
if video_path and not video.duration:
if video_path and not video.file_group.length:
# Duration was not retrieved during poster generation.
if video.ffprobe_json:
if duration := video.ffprobe_json['format'].get('duration'):
video.duration = float(duration)
video.file_group.length = float(duration)
elif (video_streams := video.get_streams_by_codec_type('video')) and 'duration' in video_streams[0]:
video.duration = float(video_streams[0]['duration'])
video.file_group.length = float(video_streams[0]['duration'])
else:
# Slowest method.
video.duration = get_video_duration(video_path)
video.file_group.length = get_video_duration(video_path)
if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES:
# Caption file was deleted, clear out old captions.
@ -482,11 +482,11 @@ async def get_statistics():
-- total videos
COUNT(v.id) AS "videos",
-- total videos downloaded over the past week/month/year
COUNT(v.id) FILTER (WHERE upload_date >= current_date - interval '1 week') AS "week",
COUNT(v.id) FILTER (WHERE upload_date >= current_date - interval '1 month') AS "month",
COUNT(v.id) FILTER (WHERE upload_date >= current_date - interval '1 year') AS "year",
COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 week') AS "week",
COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 month') AS "month",
COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 year') AS "year",
-- sum of all video lengths in seconds
COALESCE(SUM(v.duration), 0) AS "sum_duration",
COALESCE(SUM(fg.length), 0) AS "sum_duration",
-- sum of all video file sizes
COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size",
-- largest video
@ -511,9 +511,9 @@ async def get_statistics():
video
LEFT JOIN file_group fg on video.file_group_id = fg.id
WHERE
video.upload_date >= date_trunc('month', months.a)
AND video.upload_date < date_trunc('month', months.a) + interval '1 month'
AND video.upload_date IS NOT NULL
published_datetime >= date_trunc('month', months.a)
AND published_datetime < date_trunc('month', months.a) + interval '1 month'
AND published_datetime IS NOT NULL
GROUP BY
1
ORDER BY
@ -551,7 +551,7 @@ def parse_video_file_name(video_path: pathlib.Path) -> \
"""
A Video's file name can have data in it, this attempts to extract what may be there.
Example: {channel_name}_{upload_date}_{source_id}_title{ext}
Example: {channel_name}_{published_datetime}_{source_id}_title{ext}
"""
video_str = str(video_path)
if match := NAME_PARSER.match(video_str):

View file

@ -1,4 +1,3 @@
import datetime
import json
import pathlib
from pathlib import Path
@ -11,7 +10,6 @@ from sqlalchemy.orm.collections import InstrumentedList
from modules.videos.errors import UnknownVideo, UnknownChannel
from wrolpi.captions import read_captions
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task
from wrolpi.dates import now, TZDateTime
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download, download_manager
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
@ -29,13 +27,9 @@ class Video(ModelHelper, Base):
__tablename__ = 'video'
id = Column(Integer, primary_key=True)
duration = Column(Integer)
source_id = Column(String)
upload_date: datetime.datetime = Column(TZDateTime)
url = Column(String)
view_count = Column(Integer)
viewed = Column(TZDateTime)
ffprobe_json = deferred(Column(JSON))
source_id = Column(String) # The id from yt-dlp
view_count = Column(Integer) # The view count from the ChannelDownloader (or from initial download)
ffprobe_json = deferred(Column(JSON)) # Data that is fetched once from ffprobe (ffmpeg)
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
@ -75,7 +69,6 @@ class Video(ModelHelper, Base):
codec_names=codec_names,
codec_types=codec_types,
description=self.file_group.c_text or self.get_video_description(),
duration=self.duration,
id=self.id,
info_json_file=self.info_json_file,
info_json_path=self.info_json_path,
@ -83,8 +76,6 @@ class Video(ModelHelper, Base):
poster_path=self.poster_path,
source_id=self.source_id,
stem=split_path_stem_and_suffix(self.video_path)[0],
upload_date=self.upload_date.isoformat() if self.upload_date else None,
url=self.url,
video_path=self.video_path,
view_count=self.view_count,
)
@ -102,14 +93,11 @@ class Video(ModelHelper, Base):
def add_to_skip_list(self):
"""Add this video to the DownloadManager's skip list."""
if self.url:
download_manager.add_to_skip_list(self.url)
if self.file_group.url:
download_manager.add_to_skip_list(self.file_group.url)
else:
logger.warning(f'{self} cannot be added to skip list because it does not have a URL')
def set_viewed(self):
self.viewed = now()
def get_info_json(self) -> Optional[Dict]:
"""If this Video has an info_json file, return it's contents. Otherwise, return None."""
info_json_path = self.info_json_path
@ -156,16 +144,17 @@ class Video(ModelHelper, Base):
session = Session.object_session(self)
with get_db_curs() as curs:
if self.upload_date:
if self.file_group.published_datetime:
# Get videos next to this Video's upload date.
stmt = '''
WITH numbered_videos AS (
SELECT id,
ROW_NUMBER() OVER (ORDER BY upload_date ASC) AS row_number
FROM video
SELECT fg.id,
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
FROM file_group fg
LEFT OUTER JOIN video v on fg.id = v.file_group_id
WHERE
channel_id = %(channel_id)s
AND upload_date IS NOT NULL
v.channel_id = %(channel_id)s
AND fg.published_datetime IS NOT NULL
)
SELECT id
FROM numbered_videos
@ -174,7 +163,7 @@ class Video(ModelHelper, Base):
FROM numbered_videos
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
WHERE
id = %(video_id)s
id = %(fg_id)s
)
'''
else:
@ -198,7 +187,7 @@ class Video(ModelHelper, Base):
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
)
'''
curs.execute(stmt, dict(channel_id=self.channel_id, video_id=self.id))
curs.execute(stmt, dict(channel_id=self.channel_id, fg_id=self.file_group_id))
results = [i[0] for i in curs.fetchall()]

View file

@ -91,7 +91,6 @@ class Video:
poster_path: str
source_id: str
title: str
upload_date: date
video_path: str
name: str
caption_path: str
@ -108,7 +107,6 @@ class VideoWithChannel:
poster_path: str
source_id: str
title: str
upload_date: date
video_path: str
name: str
caption_path: str
@ -119,11 +117,6 @@ class VideoWithChannel:
channel = ChannelModel
@dataclass
class ChannelVideosResponse:
videos: List[Video]
@dataclass
class ExtendedVideo(Video):
info_json: dict
@ -131,7 +124,7 @@ class ExtendedVideo(Video):
@dataclass
class VideoResponse:
file = ExtendedVideo
file_group = ExtendedVideo
prev = ExtendedVideo
next = ExtendedVideo

View file

@ -228,7 +228,7 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit
assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
def test_search_videos(test_client, test_session, video_factory, assert_video_search, simple_channel, tag_factory):
def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
"""Search the Video table. This does not need to use a join with the File table."""
vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2',
@ -248,11 +248,11 @@ def test_search_videos(test_client, test_session, video_factory, assert_video_se
assert test_session.query(Video).count() == 3
assert test_session.query(FileGroup).count() == 3
assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='upload_date')
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-upload_date')
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-upload_date', limit=2)
assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-upload_date', limit=2, offset=1)
assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-upload_date', channel_id=simple_channel.id)
assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2, offset=1)
assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime', channel_id=simple_channel.id)
assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)

View file

@ -272,4 +272,4 @@ async def test_video_ffprobe_json(test_session, video_file):
video = test_session.query(Video).one()
assert video.ffprobe_json
assert video.duration
assert video.file_group.length

View file

@ -398,7 +398,7 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
video = Video.from_paths(test_session, video_file)
video.channel_id = channel.id
video.source_id = 'video 1 id'
video.url = 'https://www.youtube.com/watch?v=video_1_url'
video.file_group.url = 'https://www.youtube.com/watch?v=video_1_url'
test_session.commit()
mock_video_extract_info.return_value = example_playlist_json # Playlist info is fetched first.

View file

@ -79,7 +79,7 @@ def test_validate_video(test_directory, video_factory, image_bytes_factory, vide
assert vid1.video_path.stem == vid1.poster_path.stem
assert vid1.poster_path.suffix == '.jpg'
# File name date is assumed to be local timezone.
assert vid1.upload_date and vid1.upload_date.year == 2005
assert vid1.file_group.published_datetime and vid1.file_group.published_datetime.year == 2005
assert vid1.source_id == '1234567890'
assert vid1.file_group.title == 'The Title'

View file

@ -18,7 +18,7 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
"""
with get_db_session(commit=True) as session:
video = Video.find_by_id(video_id, session=session)
video.set_viewed()
video.file_group.set_viewed()
previous_video, next_video = video.get_surrounding_videos()
video = video.__json__()
@ -29,34 +29,33 @@ def get_video_for_app(video_id: int) -> Tuple[dict, Optional[dict], Optional[dic
VIDEO_ORDERS = {
'upload_date': 'v.upload_date ASC, LOWER(fg.primary_path) ASC',
'-upload_date': 'v.upload_date DESC NULLS LAST, LOWER(fg.primary_path) ASC',
'published_datetime': 'fg.published_datetime ASC, LOWER(fg.primary_path) ASC',
'-published_datetime': 'fg.published_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC',
'download_datetime': 'fg.download_datetime ASC, fg.published_datetime ASC, LOWER(fg.primary_path) ASC',
'-download_datetime': 'fg.download_datetime DESC NULLS LAST, fg.published_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC',
'rank': '2 DESC, LOWER(fg.primary_path) DESC',
'-rank': '2 ASC, LOWER(fg.primary_path) ASC',
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
'-size': 'fg.size DESC, LOWER(fg.primary_path) DESC',
'duration': 'duration ASC, LOWER(fg.primary_path) ASC',
'-duration': 'duration DESC, LOWER(fg.primary_path) DESC',
'viewed': 'v.viewed ASC',
'-viewed': 'v.viewed DESC',
'length': 'fg.length ASC, LOWER(fg.primary_path) ASC',
'-length': 'fg.length DESC, LOWER(fg.primary_path) DESC',
'viewed': 'fg.viewed ASC',
'-viewed': 'fg.viewed DESC',
'view_count': 'v.view_count ASC',
'-view_count': 'v.view_count DESC',
'modification_datetime': 'fg.modification_datetime ASC',
'-modification_datetime': 'fg.modification_datetime DESC',
}
NO_NULL_ORDERS = {
'viewed': 'v.viewed IS NOT NULL',
'-viewed': 'v.viewed IS NOT NULL',
'duration': 'v.duration IS NOT NULL',
'-duration': 'v.duration IS NOT NULL',
'viewed': 'fg.viewed IS NOT NULL',
'-viewed': 'fg.viewed IS NOT NULL',
'length': 'fg.length IS NOT NULL',
'-length': 'fg.length IS NOT NULL',
'size': 'fg.size IS NOT NULL',
'-size': 'fg.size IS NOT NULL',
'view_count': 'v.view_count IS NOT NULL',
'-view_count': 'v.view_count IS NOT NULL',
'modification_datetime': 'fg.modification_datetime IS NOT NULL',
'-modification_datetime': 'fg.modification_datetime IS NOT NULL',
}
JOIN_ORDERS = ('upload_date', '-upload_date', 'viewed', '-viewed', 'view_count', '-view_count', 'duration', '-duration')
JOIN_ORDERS = ('published_datetime', '-published_datetime', 'viewed', '-viewed', 'view_count', '-view_count',
'length', '-length')
DEFAULT_VIDEO_ORDER = 'rank'
VIDEO_QUERY_LIMIT = 24

View file

@ -129,7 +129,7 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
assert response.status_code == HTTPStatus.NO_CONTENT
# Video was added to skip list.
assert test_download_manager.is_skipped(vid1.url)
assert test_download_manager.is_skipped(vid1.file_group.url)
# Video was deleted.
assert test_session.query(Video).count() == 1
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
@ -138,7 +138,7 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
request, response = await test_async_client.delete(f'/api/videos/video/{vid2.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_download_manager.is_skipped(vid1.url, vid2.url)
assert test_download_manager.is_skipped(vid1.file_group.url, vid2.file_group.url)
assert test_session.query(Video).count() == 0
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False

View file

@ -86,7 +86,7 @@ async def rank_video_quality(video: Video) -> int:
rank = 0
# These are more valuable.
if video.url:
if video.file_group.url:
rank += 2
if video.source_id:
rank += 2

View file

@ -106,12 +106,12 @@ async def create_download(url: str, downloader: str, message: str):
async def download_or_ask_delete(video: Video, session: Session, message: str):
if video.url and can_be_downloaded(video.url):
await create_download(video.url, VideoDownloader.name, message)
if video.file_group.url and can_be_downloaded(video.file_group.url):
await create_download(video.file_group.url, VideoDownloader.name, message)
# Always re-download when possible.
do_delete = True
else:
logger.error(f'Video cannot be downloaded: {video.url or video}')
logger.error(f'Video cannot be downloaded: {video.file_group.url or video}')
do_delete = confirm(f'{message} Delete? (y/N)')
if do_delete:
@ -140,7 +140,7 @@ async def find_corrupt_videos(channel_id: int = None):
if not video.video_path.is_file():
await download_or_ask_delete(video, session, f'Video does not exist: {video.video_path}')
continue
if video.url:
if video.file_group.url:
for path in video.file_group.my_paths():
if not path.is_file() and path.name.endswith('.srt'):
files = [i for i in video.file_group.my_files() if i['path'] != path]
@ -148,7 +148,7 @@ async def find_corrupt_videos(channel_id: int = None):
session.commit()
logger.info(f'Removed non-existent srt: {video.video_path}')
if not all(i.is_file() for i in video.file_group.my_paths()):
await create_download(video.url, VideoDownloader.name,
await create_download(video.file_group.url, VideoDownloader.name,
f'Video is missing {path}: {video.video_path}')
continue

View file

@ -1,5 +1,4 @@
import logging
import re
from datetime import datetime, timezone, timedelta
from enum import Enum
from typing import Union

View file

@ -45,18 +45,18 @@ EBOOK_MIMETYPES = (
@dataclasses.dataclass
class EBookData:
cover: bytes = None
creator: str = None
author: str = None
text: str = None
title: str = None
ebook_path: pathlib.Path = None
def __bool__(self):
return bool(self.cover) or bool(self.creator) or bool(self.text) or bool(self.title)
return bool(self.cover) or bool(self.author) or bool(self.text) or bool(self.title)
def __json__(self):
d = {}
if self.creator:
d['creator'] = self.creator
if self.author:
d['author'] = self.author
if self.title:
d['title'] = self.title
if self.ebook_path:
@ -88,7 +88,7 @@ def extract_ebook_data(path: pathlib.Path, mimetype: str) -> Optional[EBookData]
if 'title' in value:
data.title = data.title or value['title'][0][0]
if 'creator' in value:
data.creator = data.creator or value['creator'][0][0]
data.author = data.author or value['creator'][0][0]
data.text = ''
for doc in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
@ -150,8 +150,6 @@ class EBook(ModelHelper, Base):
id = Column(Integer, primary_key=True)
size: int = Column(Integer)
creator: str = Column(String)
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
file_group: FileGroup = relationship('FileGroup')
@ -239,8 +237,8 @@ def _model_ebook(ebook: EBook) -> EBook:
if data:
# Title is a_text.
ebook.file_group.title = ebook.file_group.a_text = data.title
# Creator is b_text.
ebook.file_group.b_text = ebook.creator = data.creator
# Author is b_text.
ebook.file_group.b_text = ebook.file_group.author = data.author
# All text is d_text.
ebook.file_group.d_text = data.text
d: dict = data.__json__()

View file

@ -65,12 +65,13 @@ class FileGroup(ModelHelper, Base):
__tablename__ = 'file_group'
id: int = Column(BigInteger, primary_key=True)
# the Path of the file that can be modeled or indexed.
author = Column(String) # name of the author, maybe even a URL
data = Column(FancyJSON) # populated by the modeler
download_datetime = Column(TZDateTime) # the date WROLPi downloaded this file.
files = Column(FancyJSON, nullable=False) # populated during discovery
idempotency = Column(TZDateTime) # used to track which files need to be deleted during refresh
indexed = Column(Boolean, default=lambda: False, nullable=False) # wrolpi.files.lib.apply_indexers
length = Column(BigInteger) # video duration, article words, etc.
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
model = Column(String) # "video", "archive", "ebook", etc.
modification_datetime = Column(TZDateTime) # the modification date of the file on disk
@ -79,6 +80,8 @@ class FileGroup(ModelHelper, Base):
published_modified_datetime = Column(TZDateTime) # the date the publisher modified this file
size = Column(BigInteger, default=lambda: 0)
title = Column(String) # user-displayable title
url = Column(String) # the location where this file can be downloaded.
viewed = Column(TZDateTime) # the most recent time a User viewed this file.
tag_files: InstrumentedList = relationship('TagFile', cascade='all')
@ -103,23 +106,26 @@ class FileGroup(ModelHelper, Base):
_, suffix = split_path_stem_and_suffix(self.primary_path)
tags = sorted([i.tag.name for i in self.tag_files])
d = {
'author': self.author,
'data': self.data,
'directory': self.primary_path.parent,
'download_datetime': self.download_datetime,
'files': self.my_files(),
'id': self.id,
'key': self.primary_path,
'length': self.length,
'mimetype': self.mimetype,
'model': self.model,
'modified': self.modification_datetime or None,
'name': self.primary_path.name,
'primary_path': self.primary_path,
'published_datetime': self.published_datetime,
'published_modified_datetime': self.published_modified_datetime,
'size': self.size,
'suffix': suffix,
'tags': tags,
'title': self.title,
'key': self.primary_path,
'author': self.author,
'published_datetime': self.published_datetime,
'published_modified_datetime': self.published_modified_datetime,
'url': self.url
}
return d
@ -129,6 +135,9 @@ class FileGroup(ModelHelper, Base):
return self.title
return self.primary_path.name
def set_viewed(self):
self.viewed = now()
@optional_session
def add_tag(self, tag: Tag, session: Session = None) -> TagFile:
return tag.add_file_group_tag(self, session)

View file

@ -21,15 +21,15 @@ async def test_index(test_session, test_directory, example_epub, example_mobi):
assert ebook.file_group.title == 'WROLPi Test Book'
assert ebook.file_group.mimetype.startswith(EPUB_MIMETYPE)
assert ebook.creator == 'roland'
assert ebook.file_group.author == 'roland'
assert ebook.size == 292579
assert ebook.file_group.data == {'creator': 'roland', 'title': 'WROLPi Test Book',
assert ebook.file_group.data == {'author': 'roland', 'title': 'WROLPi Test Book',
'cover_path': test_directory / 'ebook example.jpeg',
'ebook_path': test_directory / 'ebook example.epub',
}
assert ebook.file_group.a_text, 'Book title was not indexed'
assert ebook.file_group.b_text, 'Book creator was not indexed'
assert ebook.file_group.b_text, 'Book author was not indexed'
assert ebook.file_group.d_text, 'Book text was not indexed'
# Both ebooks are assumed to be the same book, but different formats.
assert (epubs := ebook.file_group.my_files('application/epub+zip')) and len(epubs) == 1 \
@ -86,7 +86,7 @@ def test_search(test_session, test_client, example_epub):
ebook: EBook = test_session.query(EBook).one()
assert ebook.file_group.title == 'WROLPi Test Book'
assert ebook.creator == 'roland'
assert ebook.file_group.author == 'roland'
assert ebook.file_group.a_text == 'WROLPi Test Book', 'Book title was not updated'
assert ebook.file_group.d_text, 'Book was not indexed'
@ -100,7 +100,7 @@ def test_search(test_session, test_client, example_epub):
assert_dict_contains(
file_group['data'],
{'cover_path': 'ebook example.jpeg', 'ebook_path': 'ebook example.epub', 'title': 'WROLPi Test Book',
'creator': 'roland'},
'author': 'roland'},
)
# No Mobi ebook.
@ -143,7 +143,7 @@ async def test_move_ebook(test_session, test_directory, example_epub, image_file
ebook: EBook = test_session.query(EBook).one()
assert ebook.cover_path
assert_dict_contains(ebook.file_group.data, dict(creator='roland'))
assert_dict_contains(ebook.file_group.data, dict(author='roland'))
assert sorted([i['path'] for i in ebook.file_group.files]) == [
test_directory / 'ebook example.epub',
test_directory / 'ebook example.jpg',
@ -158,7 +158,7 @@ async def test_move_ebook(test_session, test_directory, example_epub, image_file
ebook: EBook = test_session.query(EBook).one()
assert ebook.file_group.primary_path == test_directory / 'new/ebook example.epub'
assert ebook.cover_path
assert_dict_contains(ebook.file_group.data, dict(creator='roland'))
assert_dict_contains(ebook.file_group.data, dict(author='roland'))
assert sorted([i['path'] for i in ebook.file_group.files]) == [
test_directory / 'new/ebook example.epub',
test_directory / 'new/ebook example.jpg',