mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-25 18:40:06 -04:00
ai: list custom GGUF files in the Manage tab (#720)
* ai: list custom GGUF files in the Manage tab A GGUF copied into ai/models/ by hand could be activated by editing ai.yaml, but never appeared in the Manage tab because the catalog endpoint only returned CDN/bundled models. The Active Model dropdown builds from downloaded models, so a custom file was unselectable. The catalog endpoint now merges any on-disk *.gguf that is not in the catalog as a `custom` tier model with its real size and no URL. It is discovered on every request, so a file appears the next time the page loads; the file refresh is not involved and does not index models. Catalog models are never duplicated and in-progress .tmp downloads are ignored. Custom rows read "On disk" and the hint explains the copy-in path, the tool-calling requirement, and the context_size fallback. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * ai: only list custom GGUFs llama-server can load; reset context on custom select Review found two gaps in the custom-model listing. Any *.gguf basename was listed and accepted, but start_llama_server.sh reads active_model back out of ai.yaml through read_config_value.sh. PyYAML dumps non-ASCII as escapes the reader prints literally, the reader strips " #..." before unquoting, and the start script's traversal guard rejects any ".." run. Those names got a 200 from the API and then "Model file does not exist" at startup. A shared is_loadable_model_name check now filters the catalog loop and rejects the name in manage_settings, so the UI never shows an activation that cannot start. Directories named *.gguf are skipped too. Selecting a custom model after a catalog model kept the previous model's explicit context_size (e.g. 16384 from the 4B) because the adopt branch was skipped when there was no catalog default. The default is now always written on model selection, so a custom model resets to null and the conservative default applies. The hint text says so instead of claiming the default is used unconditionally. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance
Roland
parent
168f194512
commit
4b3c6510b4
5 changed files with 162 additions and 6 deletions
|
|
@ -26,7 +26,9 @@ export function AiModelRow({model, onDownload}) {
|
|||
<Table.Cell>{humanFileSize(size)}</Table.Cell>
|
||||
<Table.Cell>
|
||||
{downloaded
|
||||
? <Button size='xs' icon='check' disabled role='save'>Downloaded</Button>
|
||||
? <Button size='xs' icon='check' disabled role='save'>
|
||||
{tier === 'custom' ? 'On disk' : 'Downloaded'}
|
||||
</Button>
|
||||
: <APIButton size='xs' icon='download' color='violet' onClick={() => onDownload(model)}>
|
||||
Download
|
||||
</APIButton>}
|
||||
|
|
@ -184,6 +186,11 @@ export function ManageAi() {
|
|||
<p>The AI assistant runs entirely on this WROLPi — no internet is needed after a model
|
||||
is downloaded. Download a model, select it, then enable the assistant. Monitor
|
||||
download progress on the <b>Downloads</b> page.</p>
|
||||
<p>You can also copy your own GGUF file into <b>ai/models/</b>; it will appear here as a
|
||||
<b>custom</b> model the next time this page loads. Use a plain ASCII file name without
|
||||
<b>#</b>, quotes, or <b>..</b>. The model must support tool calling (Qwen3, Llama 3.x,
|
||||
and Mistral instruct models do). Selecting a custom model resets the context size to the
|
||||
default; set <b>context_size</b> in <b>ai.yaml</b> to override it.</p>
|
||||
</InfoMessage>
|
||||
</>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -104,6 +104,24 @@ describe('ManageAi', () => {
|
|||
await waitFor(() => expect(postAiSettings).toHaveBeenCalledWith({enabled: true}));
|
||||
});
|
||||
|
||||
it('lists a custom GGUF from ai/models in the Active Model dropdown', async () => {
|
||||
const custom = {
|
||||
name: 'My-Custom-Model-Q8_0.gguf', tier: 'custom', size: 3_000_000_000, url: null,
|
||||
description: 'Custom model found in ai/models/. Not from the WROLPi catalog.',
|
||||
downloaded: true, active: false,
|
||||
};
|
||||
const fixture = catalogFixture();
|
||||
getAiCatalog.mockResolvedValue({...fixture, models: [...fixture.models, custom]});
|
||||
render(<ManageAi/>);
|
||||
await waitFor(() => expect(screen.getByText('On disk')).toBeInTheDocument());
|
||||
expect(screen.getByRole('cell', {name: 'custom'})).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole('textbox', {name: 'Active Model'}));
|
||||
const option = await screen.findByRole('option', {name: 'My-Custom-Model-Q8_0.gguf'});
|
||||
fireEvent.click(option);
|
||||
await waitFor(() => expect(postAiSettings).toHaveBeenCalledWith({active_model: 'My-Custom-Model-Q8_0.gguf'}));
|
||||
});
|
||||
|
||||
it('shows the slow-hardware warning on a Pi 4', async () => {
|
||||
getAiCatalog.mockResolvedValue(catalogFixture({slow_hardware: true}));
|
||||
render(<ManageAi/>);
|
||||
|
|
|
|||
|
|
@ -740,12 +740,27 @@ async def read_content(request: Request, file_group_id: int):
|
|||
async def manage_catalog(_: Request):
|
||||
models, source = await catalog.get_models_catalog()
|
||||
models_directory = catalog.get_models_directory()
|
||||
downloaded = {i.name for i in models_directory.glob('*.gguf')} if models_directory.is_dir() else set()
|
||||
on_disk = {i.name: i for i in models_directory.glob('*.gguf')} if models_directory.is_dir() else dict()
|
||||
disk_usage = sum(i.stat().st_size for i in models_directory.glob('*')) if models_directory.is_dir() else 0
|
||||
|
||||
config = get_ai_config()
|
||||
models = [dict(i, downloaded=i['name'] in downloaded, active=i['name'] == config.active_model)
|
||||
models = [dict(i, downloaded=i['name'] in on_disk, active=i['name'] == config.active_model)
|
||||
for i in models]
|
||||
# Any GGUF the user copied into ai/models themselves is a "custom" model. It is discovered
|
||||
# here on every request (no file refresh) so it appears in the Active Model dropdown.
|
||||
catalog_names = {i['name'] for i in models}
|
||||
for name, path in sorted(on_disk.items()):
|
||||
# Skip names start_llama_server.sh could not load (see is_loadable_model_name).
|
||||
if name not in catalog_names and path.is_file() and catalog.is_loadable_model_name(name):
|
||||
models.append(dict(
|
||||
name=name,
|
||||
tier='custom',
|
||||
url=None,
|
||||
size=path.stat().st_size,
|
||||
description='Custom model found in ai/models/. Not from the WROLPi catalog.',
|
||||
downloaded=True,
|
||||
active=name == config.active_model,
|
||||
))
|
||||
|
||||
total_ram = catalog.get_total_ram_bytes()
|
||||
return json_response(dict(
|
||||
|
|
@ -774,11 +789,16 @@ async def manage_settings(_: Request, body: schema.AIManageSettingsRequest):
|
|||
if body.active_model is not None:
|
||||
if body.active_model and not (catalog.get_models_directory() / body.active_model).is_file():
|
||||
raise ValidationError(f'Model is not downloaded: {body.active_model}')
|
||||
if body.active_model and not catalog.is_loadable_model_name(body.active_model):
|
||||
raise ValidationError(f'Model file name cannot be loaded by llama-server; rename it to plain ASCII'
|
||||
f' without #, quotes, or "..": {body.active_model}')
|
||||
config.active_model = body.active_model
|
||||
# Each model has its own context default (the 4B runs 16k, the small tier 8k); selecting
|
||||
# a model adopts it unless this request also sets context_size explicitly.
|
||||
if body.context_size is None and (default := catalog.get_model_default_context(body.active_model)):
|
||||
config.context_size = default
|
||||
# a model adopts it unless this request also sets context_size explicitly. A custom model
|
||||
# has no catalog default, so the previous model's explicit value is cleared rather than
|
||||
# carried over.
|
||||
if body.context_size is None:
|
||||
config.context_size = catalog.get_model_default_context(body.active_model)
|
||||
if body.enabled is not None:
|
||||
config.enabled = body.enabled
|
||||
if body.idle_unload_minutes is not None:
|
||||
|
|
|
|||
|
|
@ -51,6 +51,24 @@ def get_models_directory() -> Path:
|
|||
return get_media_directory() / 'ai/models'
|
||||
|
||||
|
||||
# Printable ASCII only, minus the characters the shell config reader cannot round-trip.
|
||||
# start_llama_server.sh reads active_model back out of ai.yaml with read_config_value.sh: PyYAML
|
||||
# escapes non-ASCII (\xE9), the reader strips " #..." before unquoting, and quotes/backslashes
|
||||
# survive as literal characters. The start script's traversal guard also rejects any ".." run.
|
||||
_UNLOADABLE_CHARS = set('#"\'\\/')
|
||||
|
||||
|
||||
def is_loadable_model_name(name: str) -> bool:
|
||||
"""True when llama-server can be started with this file name from ai/models.
|
||||
|
||||
Catalog names always pass; this exists for GGUFs the user copies in by hand."""
|
||||
if not name or not name.endswith('.gguf') or name != name.strip():
|
||||
return False
|
||||
if '..' in name or any(c in _UNLOADABLE_CHARS for c in name):
|
||||
return False
|
||||
return all(0x20 <= ord(c) < 0x7F for c in name)
|
||||
|
||||
|
||||
async def fetch_models_manifest(url: str = None) -> dict:
|
||||
"""Fetch the models manifest from the CDN and verify its GPG signature (map-manifest pattern)."""
|
||||
url = url or MODELS_MANIFEST_URL
|
||||
|
|
|
|||
|
|
@ -192,3 +192,96 @@ async def test_manage_settings_adopts_model_context(async_client, test_directory
|
|||
content = dict(active_model='Qwen3-4B-Instruct-2507-Q4_K_M.gguf', context_size=4_096)
|
||||
request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content))
|
||||
assert response.json['context_size'] == 4_096
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_catalog_lists_custom_models(async_client, test_directory, test_ai_config):
|
||||
"""A GGUF the user copied into ai/models (not from the catalog) is listed as a custom model so it
|
||||
can be selected in the Manage tab. It is discovered on every request; no file refresh needed."""
|
||||
models_dir = test_directory / 'ai/models'
|
||||
models_dir.mkdir(parents=True)
|
||||
(models_dir / 'My-Custom-Model-Q8_0.gguf').write_bytes(b'GGUF' * 10)
|
||||
(models_dir / 'Qwen3-1.7B-Q4_K_M.gguf').write_bytes(b'GGUF')
|
||||
(models_dir / 'partial.gguf.tmp').write_bytes(b'x') # an in-progress download is not a model
|
||||
|
||||
with mock.patch('modules.ai.catalog.fetch_models_manifest', side_effect=RuntimeError('offline')):
|
||||
request, response = await async_client.get('/api/ai/manage/catalog')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
names = [i['name'] for i in response.json['models']]
|
||||
assert names.count('Qwen3-1.7B-Q4_K_M.gguf') == 1, 'catalog model must not be duplicated'
|
||||
assert 'partial.gguf.tmp' not in names
|
||||
|
||||
custom = next(i for i in response.json['models'] if i['name'] == 'My-Custom-Model-Q8_0.gguf')
|
||||
assert custom['tier'] == 'custom'
|
||||
assert custom['downloaded'] is True
|
||||
assert custom['active'] is False
|
||||
assert custom['size'] == 40
|
||||
assert custom['url'] is None
|
||||
|
||||
# Selecting a catalog model first stores its context default in ai.yaml ...
|
||||
content = dict(active_model='Qwen3-1.7B-Q4_K_M.gguf')
|
||||
request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['context_size'] == 8_192
|
||||
|
||||
# ... and switching to the custom model clears it, since a custom model has no catalog
|
||||
# default; it must not silently keep the previous model's value.
|
||||
content = dict(active_model='My-Custom-Model-Q8_0.gguf')
|
||||
request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['active_model'] == 'My-Custom-Model-Q8_0.gguf'
|
||||
assert response.json['context_size'] is None
|
||||
assert catalog.get_effective_context_size() == 8_192 # the conservative default
|
||||
|
||||
# An explicit context_size in the same request still wins for a custom model.
|
||||
content = dict(active_model='My-Custom-Model-Q8_0.gguf', context_size=4_096)
|
||||
request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content))
|
||||
assert response.json['context_size'] == 4_096
|
||||
with mock.patch('modules.ai.catalog.fetch_models_manifest', side_effect=RuntimeError('offline')):
|
||||
request, response = await async_client.get('/api/ai/manage/catalog')
|
||||
custom = next(i for i in response.json['models'] if i['name'] == 'My-Custom-Model-Q8_0.gguf')
|
||||
assert custom['active'] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize('name,loadable', [
|
||||
('My-Custom-Model-Q8_0.gguf', True),
|
||||
('model (v2) [q4].gguf', True), # shell-safe once quoted by PyYAML
|
||||
('café.gguf', False), # PyYAML dumps \xE9; read_config_value.sh prints the escape literally
|
||||
('模型-Q4.gguf', False),
|
||||
('model #1.gguf', False), # the reader strips " #..." before unquoting
|
||||
('foo..bar.gguf', False), # start_llama_server.sh's traversal guard rejects any ".."
|
||||
('it\'s.gguf', False),
|
||||
('say "hi".gguf', False),
|
||||
('back\\slash.gguf', False),
|
||||
(' padded.gguf', False),
|
||||
('model.bin', False),
|
||||
])
|
||||
def test_is_loadable_model_name(name, loadable):
|
||||
"""Only names start_llama_server.sh can read back out of ai.yaml are loadable."""
|
||||
assert catalog.is_loadable_model_name(name) is loadable
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_catalog_skips_unloadable_custom_models(async_client, test_directory, test_ai_config):
|
||||
"""A copied-in GGUF whose name cannot round-trip through ai.yaml and the shell reader is neither
|
||||
listed nor accepted, so the UI never reports an activation that llama-server would refuse."""
|
||||
models_dir = test_directory / 'ai/models'
|
||||
models_dir.mkdir(parents=True)
|
||||
(models_dir / 'café.gguf').write_bytes(b'GGUF')
|
||||
(models_dir / 'model #1.gguf').write_bytes(b'GGUF')
|
||||
(models_dir / 'foo..bar.gguf').write_bytes(b'GGUF')
|
||||
(models_dir / 'Fine-Model.gguf').write_bytes(b'GGUF')
|
||||
(models_dir / 'a-directory.gguf').mkdir()
|
||||
|
||||
with mock.patch('modules.ai.catalog.fetch_models_manifest', side_effect=RuntimeError('offline')):
|
||||
request, response = await async_client.get('/api/ai/manage/catalog')
|
||||
names = [i['name'] for i in response.json['models']]
|
||||
assert 'Fine-Model.gguf' in names
|
||||
for bad in ('café.gguf', 'model #1.gguf', 'foo..bar.gguf', 'a-directory.gguf'):
|
||||
assert bad not in names
|
||||
|
||||
for bad in ('café.gguf', 'model #1.gguf', 'foo..bar.gguf'):
|
||||
request, response = await async_client.post('/api/ai/manage/settings',
|
||||
content=json.dumps(dict(active_model=bad)))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, bad
|
||||
assert 'cannot be loaded' in response.json['error']
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue