mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
Retag existing markdown file groups in a migration
A refresh skips a group whose files have not changed, so markdown indexed before text/markdown existed kept its old type (text/plain, or a language libmagic guessed) and that type's indexer. A migration retags those rows and marks them unindexed so apply_indexers re-indexes them with TextIndexer. The preview's suffix check is what covers such rows, not a later refresh; its comment now says so. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f1872a4d79
commit
ba76a18a57
4 changed files with 84 additions and 2 deletions
31
alembic/versions/2026_10_05_1200_markdown_mimetype.py
Normal file
31
alembic/versions/2026_10_05_1200_markdown_mimetype.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""Retag existing markdown file groups as text/markdown.
|
||||
|
||||
A refresh skips a group whose files have not changed, so markdown indexed before text/markdown
|
||||
existed would keep the type libmagic gave it (text/plain, or a programming language when the file
|
||||
holds code) and the indexer that type chose. Marking the rows unindexed lets apply_indexers
|
||||
re-index them with TextIndexer. SQLite's LIKE is case-insensitive for ASCII, so `.MD` matches.
|
||||
|
||||
Revision ID: 2026_10_05_1200
|
||||
Revises: 2026_09_28_1200
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '2026_10_05_1200'
|
||||
down_revision = '2026_09_28_1200'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.execute("""
|
||||
UPDATE file_group SET mimetype = 'text/markdown', "indexed" = 0
|
||||
WHERE (primary_path LIKE '%.md' OR primary_path LIKE '%.markdown')
|
||||
AND mimetype LIKE 'text/%'
|
||||
AND mimetype != 'text/markdown'
|
||||
""")
|
||||
|
||||
|
||||
def downgrade():
|
||||
# The original types are not recorded; text/plain is what libmagic reports for most markdown.
|
||||
op.execute("""UPDATE file_group SET mimetype = 'text/plain', "indexed" = 0 WHERE mimetype = 'text/markdown'""")
|
||||
|
|
@ -108,7 +108,8 @@ export function renderMarkdown(text, fileURL) {
|
|||
}
|
||||
|
||||
export function isMarkdownFile(mimetype, lowerPath) {
|
||||
// Files indexed before text/markdown existed stay text/plain until their next refresh.
|
||||
// By suffix too, so a markdown file whose stored type is still text/plain (or a language libmagic
|
||||
// guessed) is not shown raw; the 2026_10_05_1200 migration retags those rows, a refresh does not.
|
||||
return mimetype.startsWith('text/markdown')
|
||||
|| (mimetype.startsWith('text/') && (lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown')));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -186,7 +186,7 @@ describe('MarkdownPreview', () => {
|
|||
describe('isMarkdownFile', () => {
|
||||
test.each([
|
||||
['text/markdown', 'notes/a.md', true],
|
||||
// Files indexed before text/markdown existed are text/plain until their next refresh.
|
||||
// A stored type from before text/markdown existed; refresh does not retag unchanged files.
|
||||
['text/plain', 'notes/a.md', true],
|
||||
['text/plain', 'notes/a.markdown', true],
|
||||
['text/plain', 'notes/a.txt', false],
|
||||
|
|
|
|||
|
|
@ -211,6 +211,56 @@ def test_domain_collection_summary_migration_backfills(test_directory):
|
|||
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (1, 100)
|
||||
|
||||
|
||||
def test_markdown_mimetype_migration_retags_existing_rows(test_directory):
|
||||
"""Markdown indexed before text/markdown existed is retagged and re-indexed. A refresh skips a
|
||||
group whose files have not changed, so it would otherwise keep its old mimetype and indexer."""
|
||||
import sqlite3
|
||||
from alembic.config import Config
|
||||
from alembic import command
|
||||
from wrolpi.db import get_db_uri, get_db_file
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
(test_directory / 'config').mkdir(parents=True, exist_ok=True)
|
||||
config = Config(str(PROJECT_DIR / 'alembic.ini'))
|
||||
config.set_main_option('sqlalchemy.url', get_db_uri())
|
||||
command.upgrade(config, '2026_09_28_1200')
|
||||
|
||||
rows = (
|
||||
(1, 'notes.md', 'text/plain'),
|
||||
# libmagic reports a programming language when a markdown file holds code.
|
||||
(2, 'code.markdown', 'text/x-c'),
|
||||
(3, 'UPPER.MD', 'text/html'),
|
||||
(4, 'already.md', 'text/markdown'),
|
||||
(5, 'readme.txt', 'text/plain'),
|
||||
# Not text, so not markdown, whatever its name.
|
||||
(6, 'binary.md', 'application/octet-stream'),
|
||||
)
|
||||
db_file = get_db_file()
|
||||
with sqlite3.connect(db_file) as conn:
|
||||
for id_, name, mimetype in rows:
|
||||
stem = name.rsplit('.', 1)[0]
|
||||
conn.execute(
|
||||
'INSERT INTO file_group (id, directory, primary_path, files, stem, mimetype, indexed)'
|
||||
' VALUES (?,?,?,?,?,?,1)',
|
||||
(id_, str(test_directory), str(test_directory / name), '[]', stem, mimetype),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
command.upgrade(config, 'head')
|
||||
|
||||
with sqlite3.connect(db_file) as conn:
|
||||
result = {r[0]: (r[1], r[2]) for r in conn.execute('SELECT id, mimetype, indexed FROM file_group')}
|
||||
assert result == {
|
||||
1: ('text/markdown', 0),
|
||||
2: ('text/markdown', 0),
|
||||
3: ('text/markdown', 0),
|
||||
# Already correct, so not re-indexed.
|
||||
4: ('text/markdown', 1),
|
||||
5: ('text/plain', 1),
|
||||
6: ('application/octet-stream', 1),
|
||||
}
|
||||
|
||||
|
||||
def test_migrations_match_models(test_directory):
|
||||
"""`alembic check`: a database migrated to head matches the SQLAlchemy models (tables, columns,
|
||||
indexes). Fails when a model gains an index or column without a migration, or vice versa."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue