From 72cb9ce4ff0787b634fb97aa86285e7ceda29e23 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 3 Feb 2022 08:50:40 -0700 Subject: [PATCH] Missing Archive titles are filled in during refresh. --- modules/archive/lib.py | 9 ++++++++- modules/archive/test/test_lib.py | 19 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 85a4a3d3..67ada869 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -514,12 +514,13 @@ async def refresh_archives(): def upsert_archive(dt: str, archive_files: ArchiveFiles, session: Session): - """Get or create an Archive and it's URL/Domain. If it already exists, update it with these new files.""" + """Get or create an Archive, and it's URL/Domain. If it already exists, update it with these new files.""" # Extract the URL from the JSON. Fail if this is not possible. try: with archive_files.readability_json.open() as fh: json_contents = json.load(fh) url = json_contents['url'] + title = json_contents.get('title') except Exception as e: raise InvalidArchive() from e @@ -532,6 +533,12 @@ def upsert_archive(dt: str, archive_files: ArchiveFiles, session: Session): session.add(archive) session.flush() + if not archive.title and title: + archive.title = title + if not archive.title and archive_files.singlefile: + # As a last resort, get the title from the HTML. + archive.title = get_title_from_html(archive_files.singlefile.read_text(), url) + # Update the archive with the files that we have. archive.archive_datetime = dt archive.singlefile_path = archive_files.singlefile diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index a6402d81..548bd58f 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -147,18 +147,24 @@ def test_fills_contents_with_refresh(test_session, archive_factory, test_client) archive3 = archive_factory('example.org') archive4 = archive_factory('example.org') # has no contents + # Clear the title, this will be filled in. + archive1.title = archive2.title = archive3.title = archive4.title = None + # Title can be found in archive3's singlefile + archive3.singlefile_path.path.write_text(' some stufflast title other stuff') + contents_title_archive = [ ('foo bar qux', 'my archive', archive1), ('foo baz qux qux', 'other archive', archive2), - ('baz qux qux qux', 'archive third', archive3), + ('baz qux qux qux', None, archive3), ] - for contents, title, archive in contents_title_archive: + for contents, json_title, archive in contents_title_archive: with archive.readability_txt_path.path.open('wt') as fh: - archive.title = title fh.write(contents) + if json_title: + archive.readability_json_path.path.write_text(json.dumps({'title': json_title, 'url': archive.url})) test_session.commit() - # Contents are empty. + # Contents are empty. Archives are all missing the title. assert not archive1.contents assert not archive2.contents assert not archive3.contents @@ -173,6 +179,11 @@ def test_fills_contents_with_refresh(test_session, archive_factory, test_client) assert archive3.contents assert not archive4.contents + # Missing title is filled. + assert archive1.title == 'my archive' + assert archive2.title == 'other archive' + assert archive3.title == 'last title' # from json + def test_delete_archive(test_session, archive_factory): archive1 = archive_factory('example.com', 'https://example.com/1')