rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Missing Archive titles are filled in during refresh.

Browse source
This commit is contained in:
Roland 2022-02-03 08:50:40 -07:00
parent 01bfe4de97
commit 72cb9ce4ff
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
2 changed files with 23 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

9
modules/archive/lib.py
View file

@ -514,12 +514,13 @@ async def refresh_archives():
def upsert_archive(dt: str, archive_files: ArchiveFiles, session: Session):
"""Get or create an Archive and it's URL/Domain. If it already exists, update it with these new files."""
"""Get or create an Archive, and it's URL/Domain. If it already exists, update it with these new files."""
# Extract the URL from the JSON. Fail if this is not possible.
try:
with archive_files.readability_json.open() as fh:
json_contents = json.load(fh)
url = json_contents['url']
title = json_contents.get('title')
except Exception as e:
raise InvalidArchive() from e
@ -532,6 +533,12 @@ def upsert_archive(dt: str, archive_files: ArchiveFiles, session: Session):
session.add(archive)
session.flush()
if not archive.title and title:
archive.title = title
if not archive.title and archive_files.singlefile:
# As a last resort, get the title from the HTML.
archive.title = get_title_from_html(archive_files.singlefile.read_text(), url)
# Update the archive with the files that we have.
archive.archive_datetime = dt
archive.singlefile_path = archive_files.singlefile

19
modules/archive/test/test_lib.py
View file

@ -147,18 +147,24 @@ def test_fills_contents_with_refresh(test_session, archive_factory, test_client)
archive3 = archive_factory('example.org')
archive4 = archive_factory('example.org') # has no contents
# Clear the title, this will be filled in.
archive1.title = archive2.title = archive3.title = archive4.title = None
# Title can be found in archive3's singlefile
archive3.singlefile_path.path.write_text('<html> some stuff<title>last title</title> other stuff</html>')
contents_title_archive = [
('foo bar qux', 'my archive', archive1),
('foo baz qux qux', 'other archive', archive2),
('baz qux qux qux', 'archive third', archive3),
('baz qux qux qux', None, archive3),
]
for contents, title, archive in contents_title_archive:
for contents, json_title, archive in contents_title_archive:
with archive.readability_txt_path.path.open('wt') as fh:
archive.title = title
fh.write(contents)
if json_title:
archive.readability_json_path.path.write_text(json.dumps({'title': json_title, 'url': archive.url}))
test_session.commit()
# Contents are empty.
# Contents are empty. Archives are all missing the title.
assert not archive1.contents
assert not archive2.contents
assert not archive3.contents
@ -173,6 +179,11 @@ def test_fills_contents_with_refresh(test_session, archive_factory, test_client)
assert archive3.contents
assert not archive4.contents
# Missing title is filled.
assert archive1.title == 'my archive'
assert archive2.title == 'other archive'
assert archive3.title == 'last title' # from json
def test_delete_archive(test_session, archive_factory):
archive1 = archive_factory('example.com', 'https://example.com/1')