diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 85a4a3d3..67ada869 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -514,12 +514,13 @@ async def refresh_archives(): def upsert_archive(dt: str, archive_files: ArchiveFiles, session: Session): - """Get or create an Archive and it's URL/Domain. If it already exists, update it with these new files.""" + """Get or create an Archive, and it's URL/Domain. If it already exists, update it with these new files.""" # Extract the URL from the JSON. Fail if this is not possible. try: with archive_files.readability_json.open() as fh: json_contents = json.load(fh) url = json_contents['url'] + title = json_contents.get('title') except Exception as e: raise InvalidArchive() from e @@ -532,6 +533,12 @@ def upsert_archive(dt: str, archive_files: ArchiveFiles, session: Session): session.add(archive) session.flush() + if not archive.title and title: + archive.title = title + if not archive.title and archive_files.singlefile: + # As a last resort, get the title from the HTML. + archive.title = get_title_from_html(archive_files.singlefile.read_text(), url) + # Update the archive with the files that we have. archive.archive_datetime = dt archive.singlefile_path = archive_files.singlefile diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index a6402d81..548bd58f 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -147,18 +147,24 @@ def test_fills_contents_with_refresh(test_session, archive_factory, test_client) archive3 = archive_factory('example.org') archive4 = archive_factory('example.org') # has no contents + # Clear the title, this will be filled in. + archive1.title = archive2.title = archive3.title = archive4.title = None + # Title can be found in archive3's singlefile + archive3.singlefile_path.path.write_text(' some stuff