Retag existing markdown file groups in a migration

A refresh skips a group whose files have not changed, so markdown indexed
before text/markdown existed kept its old type (text/plain, or a language
libmagic guessed) and that type's indexer.  A migration retags those rows and
marks them unindexed so apply_indexers re-indexes them with TextIndexer.

The preview's suffix check is what covers such rows, not a later refresh;
its comment now says so.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-05 13:19:34 -06:00
parent f1872a4d79
commit ba76a18a57
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 84 additions and 2 deletions

View file

@ -0,0 +1,31 @@
"""Retag existing markdown file groups as text/markdown.
A refresh skips a group whose files have not changed, so markdown indexed before text/markdown
existed would keep the type libmagic gave it (text/plain, or a programming language when the file
holds code) and the indexer that type chose. Marking the rows unindexed lets apply_indexers
re-index them with TextIndexer. SQLite's LIKE is case-insensitive for ASCII, so `.MD` matches.
Revision ID: 2026_10_05_1200
Revises: 2026_09_28_1200
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = '2026_10_05_1200'
down_revision = '2026_09_28_1200'
branch_labels = None
depends_on = None
def upgrade():
op.execute("""
UPDATE file_group SET mimetype = 'text/markdown', "indexed" = 0
WHERE (primary_path LIKE '%.md' OR primary_path LIKE '%.markdown')
AND mimetype LIKE 'text/%'
AND mimetype != 'text/markdown'
""")
def downgrade():
# The original types are not recorded; text/plain is what libmagic reports for most markdown.
op.execute("""UPDATE file_group SET mimetype = 'text/plain', "indexed" = 0 WHERE mimetype = 'text/markdown'""")

View file

@ -108,7 +108,8 @@ export function renderMarkdown(text, fileURL) {
}
export function isMarkdownFile(mimetype, lowerPath) {
// Files indexed before text/markdown existed stay text/plain until their next refresh.
// By suffix too, so a markdown file whose stored type is still text/plain (or a language libmagic
// guessed) is not shown raw; the 2026_10_05_1200 migration retags those rows, a refresh does not.
return mimetype.startsWith('text/markdown')
|| (mimetype.startsWith('text/') && (lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown')));
}

View file

@ -186,7 +186,7 @@ describe('MarkdownPreview', () => {
describe('isMarkdownFile', () => {
test.each([
['text/markdown', 'notes/a.md', true],
// Files indexed before text/markdown existed are text/plain until their next refresh.
// A stored type from before text/markdown existed; refresh does not retag unchanged files.
['text/plain', 'notes/a.md', true],
['text/plain', 'notes/a.markdown', true],
['text/plain', 'notes/a.txt', false],

View file

@ -211,6 +211,56 @@ def test_domain_collection_summary_migration_backfills(test_directory):
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (1, 100)
def test_markdown_mimetype_migration_retags_existing_rows(test_directory):
"""Markdown indexed before text/markdown existed is retagged and re-indexed. A refresh skips a
group whose files have not changed, so it would otherwise keep its old mimetype and indexer."""
import sqlite3
from alembic.config import Config
from alembic import command
from wrolpi.db import get_db_uri, get_db_file
from wrolpi.vars import PROJECT_DIR
(test_directory / 'config').mkdir(parents=True, exist_ok=True)
config = Config(str(PROJECT_DIR / 'alembic.ini'))
config.set_main_option('sqlalchemy.url', get_db_uri())
command.upgrade(config, '2026_09_28_1200')
rows = (
(1, 'notes.md', 'text/plain'),
# libmagic reports a programming language when a markdown file holds code.
(2, 'code.markdown', 'text/x-c'),
(3, 'UPPER.MD', 'text/html'),
(4, 'already.md', 'text/markdown'),
(5, 'readme.txt', 'text/plain'),
# Not text, so not markdown, whatever its name.
(6, 'binary.md', 'application/octet-stream'),
)
db_file = get_db_file()
with sqlite3.connect(db_file) as conn:
for id_, name, mimetype in rows:
stem = name.rsplit('.', 1)[0]
conn.execute(
'INSERT INTO file_group (id, directory, primary_path, files, stem, mimetype, indexed)'
' VALUES (?,?,?,?,?,?,1)',
(id_, str(test_directory), str(test_directory / name), '[]', stem, mimetype),
)
conn.commit()
command.upgrade(config, 'head')
with sqlite3.connect(db_file) as conn:
result = {r[0]: (r[1], r[2]) for r in conn.execute('SELECT id, mimetype, indexed FROM file_group')}
assert result == {
1: ('text/markdown', 0),
2: ('text/markdown', 0),
3: ('text/markdown', 0),
# Already correct, so not re-indexed.
4: ('text/markdown', 1),
5: ('text/plain', 1),
6: ('application/octet-stream', 1),
}
def test_migrations_match_models(test_directory):
"""`alembic check`: a database migrated to head matches the SQLAlchemy models (tables, columns,
indexes). Fails when a model gains an index or column without a migration, or vice versa."""