wrolpi/modules/repos/test/test_api.py

import json
from http import HTTPStatus

import pytest

from modules.repos import lib
from modules.repos.models import Repository
from wrolpi.conftest import await_file_worker, production_like_sessions, write_lock_held_briefly
from wrolpi.downloader import Download, DownloadFrequency, DownloadStatus
from wrolpi.files.models import FileGroup


@pytest.mark.asyncio
async def test_repos_crud(test_session, test_directory, async_client, test_wrolpi_config):
    body = dict(url='https://github.com/kiwix/kiwix-tools', tag_name='software', frequency=DownloadFrequency.daily)
    request, response = await async_client.post('/api/repos', content=json.dumps(body))
    assert response.status_code == HTTPStatus.CREATED, response.json
    repo = response.json['repo']
    assert repo['name'] == 'kiwix-tools'
    assert repo['tag_name'] == 'software'
    assert repo['directory'] == 'repos/software/kiwix-tools'
    assert repo['mode'] == 'full'
    assert repo['frequency'] == DownloadFrequency.daily
    assert repo['download_status'] == DownloadStatus.new
    assert repo['location'] == f'/repos/{repo["id"]}'
    repo_id = repo['id']

    request, response = await async_client.post('/api/repos', content=json.dumps(dict(url='https://github.com/a/b')))
    assert response.status_code == HTTPStatus.CREATED

    request, response = await async_client.get('/api/repos')
    assert response.status_code == HTTPStatus.OK
    assert [i['name'] for i in response.json['repos']] == ['b', 'kiwix-tools']

    body = dict(description='Kiwix tools', mode='snapshot', branch='dev', frequency=DownloadFrequency.weekly)
    request, response = await async_client.put(f'/api/repos/{repo_id}', content=json.dumps(body))
    assert response.status_code == HTTPStatus.OK, response.json
    request, response = await async_client.get(f'/api/repos/{repo_id}')
    repo = response.json['repo']
    assert (repo['description'], repo['mode'], repo['branch'], repo['frequency']) == \
           ('Kiwix tools', 'snapshot', 'dev', DownloadFrequency.weekly)

    request, response = await async_client.delete(f'/api/repos/{repo_id}')
    assert response.status_code == HTTPStatus.NO_CONTENT
    request, response = await async_client.get(f'/api/repos/{repo_id}')
    assert response.status_code == HTTPStatus.NOT_FOUND
    assert response.json['code'] == 'UNKNOWN_REPO'


@pytest.mark.asyncio
async def test_create_repo_errors(test_session, async_client, test_wrolpi_config):
    request, response = await async_client.post('/api/repos', content=json.dumps(dict(url='git@github.com:a/b')))
    assert response.status_code == HTTPStatus.BAD_REQUEST
    assert response.json['code'] == 'INVALID_REPO'

    request, response = await async_client.post('/api/repos', content=json.dumps(dict(url='https://github.com/a/b')))
    assert response.status_code == HTTPStatus.CREATED
    request, response = await async_client.post('/api/repos',
                                                content=json.dumps(dict(url='https://github.com/a/b.git')))
    assert response.status_code == HTTPStatus.CONFLICT
    assert response.json['code'] == 'REPO_CONFLICT'


@pytest.mark.asyncio
async def test_repo_tree_and_update(test_session, async_client, git_remote_factory, repos_download_manager):
    remote = git_remote_factory(files={'README.md': '# Example', 'docs/guide.md': 'guide'})
    repo = lib.create_repository(test_session, remote.url)
    await repos_download_manager.wait_for_all_downloads()

    request, response = await async_client.get(f'/api/repos/{repo.id}/tree')
    assert response.status_code == HTTPStatus.OK
    assert response.json['readme_path'] == 'README.md'
    assert [i['path'] for i in response.json['entries']] == ['docs', 'README.md']

    request, response = await async_client.get(f'/api/repos/{repo.id}/tree?path=docs')
    assert [i['path'] for i in response.json['entries']] == ['docs/guide.md']

    request, response = await async_client.get(f'/api/repos/{repo.id}/tree?path=../..')
    assert response.status_code == HTTPStatus.BAD_REQUEST

    # "Update now" renews the download.
    new_head = remote.commit({'new.txt': 'new'}, 'New')
    request, response = await async_client.post(f'/api/repos/{repo.id}/update')
    assert response.status_code == HTTPStatus.NO_CONTENT
    await repos_download_manager.wait_for_all_downloads()
    test_session.expire_all()
    assert test_session.query(Repository).one().head_sha == new_head


@pytest.mark.asyncio
async def test_tag_repo_moves_clone(test_session, test_directory, async_client, git_remote_factory,
                                    repos_download_manager, await_background_tasks):
    """Tagging a Repo (through the collections API) moves the whole clone, including `.git`.  Its files are not
    indexed (repos/ is ignored), even ones the file mover could not group."""
    remote = git_remote_factory(files={'README.md': '# Example', 'web/.npmrc': 'a', 'web/.gitignore': 'b'})
    repo = lib.create_repository(test_session, remote.url)
    await repos_download_manager.wait_for_all_downloads()
    old_directory = test_directory / 'repos/example'
    assert (old_directory / '.git').is_dir()

    body = dict(tag_name='software', directory='repos/software/example')
    request, response = await async_client.post(f'/api/collections/{repo.collection_id}/tag',
                                                content=json.dumps(body))
    assert response.status_code == HTTPStatus.OK, response.json
    await await_background_tasks()
    test_session.expire_all()

    new_directory = test_directory / 'repos/software/example'
    assert repo.directory == new_directory
    assert repo.tag_name == 'software'
    assert (new_directory / '.git').is_dir()
    assert (new_directory / 'README.md').is_file()
    assert (new_directory / 'web/.npmrc').is_file()
    assert not old_directory.exists()
    assert test_session.query(Download).one().destination == new_directory
    assert test_session.query(FileGroup).count() == 0

    # The moved clone still updates.
    new_head = remote.commit({'new.txt': 'new'}, 'New')
    download = test_session.query(Download).one()
    download.renew()
    test_session.commit()
    await repos_download_manager.wait_for_all_downloads()
    test_session.expire_all()
    assert repo.head_sha == new_head
    assert (new_directory / 'new.txt').is_file()


@pytest.mark.asyncio
async def test_tag_repo_in_custom_directory(test_session, test_directory, async_client, git_remote_factory,
                                            repos_download_manager, await_background_tasks, refresh_files):
    """A Repo in a directory which is not ignored is indexed where it is moved to, and not where it was."""
    remote = git_remote_factory(files={'README.md': '# Example', 'guide.md': 'A guide'})
    repo = lib.create_repository(test_session, remote.url, directory='projects/example')
    await repos_download_manager.wait_for_all_downloads()
    await refresh_files()
    old_directory = test_directory / 'projects/example'
    assert test_session.query(FileGroup).filter(FileGroup.primary_path == str(old_directory / 'guide.md')).count()

    body = dict(tag_name='software', directory='projects/moved')
    request, response = await async_client.post(f'/api/collections/{repo.collection_id}/tag',
                                                content=json.dumps(body))
    assert response.status_code == HTTPStatus.OK, response.json
    await await_background_tasks()
    await await_file_worker()
    test_session.expire_all()

    new_directory = test_directory / 'projects/moved'
    assert repo.directory == new_directory
    assert (new_directory / '.git').is_dir()
    assert not old_directory.exists()
    paths = {i.primary_path for i in test_session.query(FileGroup)}
    assert new_directory / 'guide.md' in paths
    assert not [i for i in paths if old_directory in i.parents]


@pytest.mark.parametrize('directory', ['../escape/example', 'repos/../../escape/example', '{media}/../escape/example',
                                       'linked/example'])
@pytest.mark.asyncio
async def test_tag_repo_cannot_leave_media_directory(test_session, test_directory, async_client, git_remote_factory,
                                                     repos_download_manager, await_background_tasks,
                                                     tmp_path_factory, directory):
    """Tagging cannot move a Repo's clone out of the media directory, by `..` or through a link."""
    remote = git_remote_factory()
    repo = lib.create_repository(test_session, remote.url)
    await repos_download_manager.wait_for_all_downloads()
    outside = tmp_path_factory.mktemp('outside')
    (test_directory / 'linked').symlink_to(outside)

    body = dict(tag_name='software', directory=directory.format(media=test_directory))
    request, response = await async_client.post(f'/api/collections/{repo.collection_id}/tag',
                                                content=json.dumps(body))
    await await_background_tasks()
    test_session.expire_all()

    assert (test_directory / 'repos/example/.git').is_dir()
    assert repo.directory == test_directory / 'repos/example'
    assert not (test_directory.parent / 'escape').exists()
    assert list(outside.iterdir()) == []


@pytest.mark.asyncio
async def test_repos_destination_setting(async_client, test_wrolpi_config):
    request, response = await async_client.get('/api/settings')
    assert response.json['repos_destination'] == 'repos/%(repo_tag)s/%(repo_name)s'

    body = dict(repos_destination='/repos')
    request, response = await async_client.patch('/api/settings', content=json.dumps(body))
    assert response.status_code == HTTPStatus.BAD_REQUEST

    body = dict(repos_destination='repos/%(repo_owner)s/%(repo_name)s')
    request, response = await async_client.patch('/api/settings', content=json.dumps(body))
    assert response.status_code == HTTPStatus.NO_CONTENT
    request, response = await async_client.get('/api/settings')
    assert response.json['repos_destination'] == 'repos/%(repo_owner)s/%(repo_name)s'


@pytest.mark.asyncio
async def test_repo_changes_wait_for_config_import(test_session, test_directory, async_client, git_remote_factory,
                                                   repos_download_manager, test_wrolpi_config):
    """Until repos.yaml is imported (e.g. while WROLPi starts), Repos cannot change; the import would undo it."""
    remote = git_remote_factory()
    other = git_remote_factory('other')
    repo = lib.create_repository(test_session, remote.url)
    await repos_download_manager.wait_for_all_downloads()
    config = lib.get_repos_config()
    config.dump_config()
    assert config.get_file().is_file()
    config.successful_import = False

    requests = [
        ('post', '/api/repos', dict(url=other.url)),
        ('post', '/api/repos/import', dict(directory='repos/example', confirm=True)),
        ('put', f'/api/repos/{repo.id}', dict(description='changed')),
        ('delete', f'/api/repos/{repo.id}', None),
        ('post', f'/api/collections/{repo.collection_id}/tag', dict(tag_name='software',
                                                                     directory='repos/software/example')),
        ('put', f'/api/collections/{repo.collection_id}', dict(description='changed')),
        ('delete', f'/api/collections/{repo.collection_id}', None),
    ]
    for method, url, body in requests:
        kwargs = dict(content=json.dumps(body)) if body else {}
        request, response = await getattr(async_client, method)(url, **kwargs)
        assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE, (url, response.json)
        assert response.json['code'] == 'CONFIG_NOT_IMPORTED'
    test_session.expire_all()
    assert test_session.query(Repository).count() == 1
    assert repo.collection.description is None
    assert repo.tag_name is None
    assert (test_directory / 'repos/example/.git').is_dir()

    # Once imported, Repos can change again.
    lib.import_repos_config()
    request, response = await async_client.put(f'/api/repos/{repo.id}', content=json.dumps(dict(description='x')))
    assert response.status_code == HTTPStatus.OK, response.json


@pytest.mark.asyncio
@pytest.mark.parametrize('step', ['create', 'change', 'delete', 'update_now'])
async def test_repo_writes_wait_for_a_concurrent_writer(test_session, async_client, test_wrolpi_config,
                                                        git_remote_factory, repos_download_manager, step):
    """These requests read before they write, so each must begin as a writer: SQLite fails a deferred
    transaction's lock upgrade instantly ("database is locked") if another connection wrote meanwhile."""
    remote = git_remote_factory()
    repo_id = None
    if step != 'create':
        repo_id = lib.create_repository(test_session, remote.url).id
        await repos_download_manager.wait_for_all_downloads()
    test_session.commit()
    db_file = test_session.get_bind().url.database

    with production_like_sessions(test_session) as maker:
        # Held past any slow work a request does before it writes.
        with write_lock_held_briefly(db_file, seconds=3):
            if step == 'create':
                _, response = await async_client.post('/api/repos', content=json.dumps(dict(url=remote.url)))
            elif step == 'change':
                _, response = await async_client.put(f'/api/repos/{repo_id}',
                                                     content=json.dumps(dict(description='Waited')))
            elif step == 'delete':
                _, response = await async_client.delete(f'/api/repos/{repo_id}')
            else:
                _, response = await async_client.post(f'/api/repos/{repo_id}/update')
        assert response.status_code in (HTTPStatus.OK, HTTPStatus.CREATED, HTTPStatus.NO_CONTENT), response.json

        with maker() as session:
            repos = session.query(Repository).all()
            if step == 'delete':
                assert repos == []
            elif step == 'change':
                assert repos[0].collection.description == 'Waited'
            else:
                assert len(repos) == 1
    await repos_download_manager.wait_for_all_downloads()