From ba76a18a57519a10d39494acfb0917049c6fc620 Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 5 Oct 2026 13:19:34 -0600 Subject: [PATCH] Retag existing markdown file groups in a migration A refresh skips a group whose files have not changed, so markdown indexed before text/markdown existed kept its old type (text/plain, or a language libmagic guessed) and that type's indexer. A migration retags those rows and marks them unindexed so apply_indexers re-indexes them with TextIndexer. The preview's suffix check is what covers such rows, not a later refresh; its comment now says so. Co-Authored-By: Claude Opus 5.5 --- .../2026_10_05_1200_markdown_mimetype.py | 31 ++++++++++++ app/src/components/MarkdownPreview.js | 3 +- app/src/components/MarkdownPreview.test.js | 2 +- wrolpi/test/test_db_bootstrap.py | 50 +++++++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/2026_10_05_1200_markdown_mimetype.py diff --git a/alembic/versions/2026_10_05_1200_markdown_mimetype.py b/alembic/versions/2026_10_05_1200_markdown_mimetype.py new file mode 100644 index 00000000..02300ea4 --- /dev/null +++ b/alembic/versions/2026_10_05_1200_markdown_mimetype.py @@ -0,0 +1,31 @@ +"""Retag existing markdown file groups as text/markdown. + +A refresh skips a group whose files have not changed, so markdown indexed before text/markdown +existed would keep the type libmagic gave it (text/plain, or a programming language when the file +holds code) and the indexer that type chose. Marking the rows unindexed lets apply_indexers +re-index them with TextIndexer. SQLite's LIKE is case-insensitive for ASCII, so `.MD` matches. + +Revision ID: 2026_10_05_1200 +Revises: 2026_09_28_1200 +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_10_05_1200' +down_revision = '2026_09_28_1200' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute(""" + UPDATE file_group SET mimetype = 'text/markdown', "indexed" = 0 + WHERE (primary_path LIKE '%.md' OR primary_path LIKE '%.markdown') + AND mimetype LIKE 'text/%' + AND mimetype != 'text/markdown' + """) + + +def downgrade(): + # The original types are not recorded; text/plain is what libmagic reports for most markdown. + op.execute("""UPDATE file_group SET mimetype = 'text/plain', "indexed" = 0 WHERE mimetype = 'text/markdown'""") diff --git a/app/src/components/MarkdownPreview.js b/app/src/components/MarkdownPreview.js index b668c2ec..5d7e1848 100644 --- a/app/src/components/MarkdownPreview.js +++ b/app/src/components/MarkdownPreview.js @@ -108,7 +108,8 @@ export function renderMarkdown(text, fileURL) { } export function isMarkdownFile(mimetype, lowerPath) { - // Files indexed before text/markdown existed stay text/plain until their next refresh. + // By suffix too, so a markdown file whose stored type is still text/plain (or a language libmagic + // guessed) is not shown raw; the 2026_10_05_1200 migration retags those rows, a refresh does not. return mimetype.startsWith('text/markdown') || (mimetype.startsWith('text/') && (lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown'))); } diff --git a/app/src/components/MarkdownPreview.test.js b/app/src/components/MarkdownPreview.test.js index 63f3d254..270e09a8 100644 --- a/app/src/components/MarkdownPreview.test.js +++ b/app/src/components/MarkdownPreview.test.js @@ -186,7 +186,7 @@ describe('MarkdownPreview', () => { describe('isMarkdownFile', () => { test.each([ ['text/markdown', 'notes/a.md', true], - // Files indexed before text/markdown existed are text/plain until their next refresh. + // A stored type from before text/markdown existed; refresh does not retag unchanged files. ['text/plain', 'notes/a.md', true], ['text/plain', 'notes/a.markdown', true], ['text/plain', 'notes/a.txt', false], diff --git a/wrolpi/test/test_db_bootstrap.py b/wrolpi/test/test_db_bootstrap.py index 681df056..78d511e0 100644 --- a/wrolpi/test/test_db_bootstrap.py +++ b/wrolpi/test/test_db_bootstrap.py @@ -211,6 +211,56 @@ def test_domain_collection_summary_migration_backfills(test_directory): assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (1, 100) +def test_markdown_mimetype_migration_retags_existing_rows(test_directory): + """Markdown indexed before text/markdown existed is retagged and re-indexed. A refresh skips a + group whose files have not changed, so it would otherwise keep its old mimetype and indexer.""" + import sqlite3 + from alembic.config import Config + from alembic import command + from wrolpi.db import get_db_uri, get_db_file + from wrolpi.vars import PROJECT_DIR + + (test_directory / 'config').mkdir(parents=True, exist_ok=True) + config = Config(str(PROJECT_DIR / 'alembic.ini')) + config.set_main_option('sqlalchemy.url', get_db_uri()) + command.upgrade(config, '2026_09_28_1200') + + rows = ( + (1, 'notes.md', 'text/plain'), + # libmagic reports a programming language when a markdown file holds code. + (2, 'code.markdown', 'text/x-c'), + (3, 'UPPER.MD', 'text/html'), + (4, 'already.md', 'text/markdown'), + (5, 'readme.txt', 'text/plain'), + # Not text, so not markdown, whatever its name. + (6, 'binary.md', 'application/octet-stream'), + ) + db_file = get_db_file() + with sqlite3.connect(db_file) as conn: + for id_, name, mimetype in rows: + stem = name.rsplit('.', 1)[0] + conn.execute( + 'INSERT INTO file_group (id, directory, primary_path, files, stem, mimetype, indexed)' + ' VALUES (?,?,?,?,?,?,1)', + (id_, str(test_directory), str(test_directory / name), '[]', stem, mimetype), + ) + conn.commit() + + command.upgrade(config, 'head') + + with sqlite3.connect(db_file) as conn: + result = {r[0]: (r[1], r[2]) for r in conn.execute('SELECT id, mimetype, indexed FROM file_group')} + assert result == { + 1: ('text/markdown', 0), + 2: ('text/markdown', 0), + 3: ('text/markdown', 0), + # Already correct, so not re-indexed. + 4: ('text/markdown', 1), + 5: ('text/plain', 1), + 6: ('application/octet-stream', 1), + } + + def test_migrations_match_models(test_directory): """`alembic check`: a database migrated to head matches the SQLAlchemy models (tables, columns, indexes). Fails when a model gains an index or column without a migration, or vice versa."""