feat: add ability to run build process to remote instances (#118)

This commit is contained in:
2023-12-13 15:38:51 +02:00
committed by GitHub
parent f2f6f6df70
commit 5ddc08fce7
39 changed files with 1062 additions and 73 deletions

View File

@ -197,33 +197,34 @@ def test_update(application_repository: ApplicationRepository, package_ahriman:
paths = [package.filepath for package in package_ahriman.packages.values()]
tree = Tree([Leaf(package_ahriman)])
mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
resolve_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.partition",
return_value=tree.levels())
mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=paths)
build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=result)
update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update", return_value=result)
build_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.update",
return_value=result)
update_mock = mocker.patch("ahriman.core.repository.Repository.process_update", return_value=result)
on_result_mock = mocker.patch(
"ahriman.application.application.application_repository.ApplicationRepository.on_result")
application_repository.update([package_ahriman], Packagers("username"), bump_pkgrel=True)
resolve_mock.assert_called_once_with([package_ahriman])
build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True)
update_mock.assert_has_calls([
MockCall(paths, Packagers("username")),
MockCall(paths, Packagers("username")),
])
update_mock.assert_called_once_with(paths, Packagers("username"))
on_result_mock.assert_has_calls([MockCall(result), MockCall(result)])
def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package,
def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package, result: Result,
mocker: MockerFixture) -> None:
"""
must skip updating repository if no packages supplied
"""
tree = Tree([Leaf(package_ahriman)])
mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[])
mocker.patch("ahriman.core.repository.executor.Executor.process_build")
update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update")
mocker.patch("ahriman.application.application.workers.Updater.partition", return_value=tree.levels())
mocker.patch("ahriman.core.repository.Repository.packages_built", return_value=[])
mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.update", return_value=result)
mocker.patch("ahriman.application.application.application_repository.ApplicationRepository.on_result")
update_mock = mocker.patch("ahriman.core.repository.Repository.process_update")
application_repository.update([package_ahriman])
update_mock.assert_not_called()

View File

@ -0,0 +1,48 @@
import pytest
from ahriman.application.application.workers import Updater
from ahriman.application.application.workers.local_updater import LocalUpdater
from ahriman.application.application.workers.remote_updater import RemoteUpdater
from ahriman.core.configuration import Configuration
from ahriman.core.repository import Repository
from ahriman.models.worker import Worker
@pytest.fixture
def local_updater(repository: Repository) -> LocalUpdater:
"""
local updater fixture
Args:
repository(Repository): repository fixture
Returns:
LocalUpdater: local updater test instance
"""
return LocalUpdater(repository)
@pytest.fixture
def remote_updater(configuration: Configuration) -> RemoteUpdater:
"""
local updater fixture
Args:
configuration(Configuration): configuration fixture
Returns:
RemoteUpdater: remote updater test instance
"""
_, repository_id = configuration.check_loaded()
return RemoteUpdater([Worker("remote1"), Worker("remote2")], repository_id, configuration)
@pytest.fixture
def updater() -> Updater:
"""
empty updater fixture
Returns:
Updater: empty updater test instance
"""
return Updater()

View File

@ -0,0 +1,30 @@
from pytest_mock import MockerFixture
from ahriman.application.application.workers.local_updater import LocalUpdater
from ahriman.models.package import Package
from ahriman.models.packagers import Packagers
from ahriman.models.result import Result
def test_partition(local_updater: LocalUpdater, mocker: MockerFixture) -> None:
"""
must partition as tree resolution
"""
resolve_mock = mocker.patch("ahriman.core.tree.Tree.resolve")
local_updater.partition([])
resolve_mock.assert_called_once_with([])
def test_update(local_updater: LocalUpdater, package_ahriman: Package, result: Result,
mocker: MockerFixture) -> None:
"""
must process package updates
"""
paths = [package.filepath for package in package_ahriman.packages.values()]
mocker.patch("ahriman.core.repository.Repository.packages_built", return_value=paths)
build_mock = mocker.patch("ahriman.core.repository.Repository.process_build", return_value=result)
update_mock = mocker.patch("ahriman.core.repository.Repository.process_update", return_value=result)
assert local_updater.update([package_ahriman], Packagers("username"), bump_pkgrel=True) == result
build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True)
update_mock.assert_called_once_with(paths, Packagers("username"))

View File

@ -0,0 +1,84 @@
from pytest_mock import MockerFixture
from ahriman.application.application.workers.remote_updater import RemoteUpdater
from ahriman.core.http import SyncAhrimanClient
from ahriman.models.package import Package
from ahriman.models.packagers import Packagers
from ahriman.models.result import Result
def test_clients(remote_updater: RemoteUpdater) -> None:
"""
must return map of clients
"""
worker = remote_updater.workers[0]
client = SyncAhrimanClient()
remote_updater._clients.append((worker, client))
assert remote_updater.clients == {worker: client}
def test_update_url(remote_updater: RemoteUpdater) -> None:
"""
must generate update url correctly
"""
worker = remote_updater.workers[0]
assert remote_updater._update_url(worker).startswith(worker.address)
assert remote_updater._update_url(worker).endswith("/api/v1/service/add")
def test_next_worker(remote_updater: RemoteUpdater) -> None:
"""
must return next not used worker
"""
assert remote_updater.next_worker()[0] == remote_updater.workers[0]
assert len(remote_updater.clients) == 1
assert remote_updater.workers[0] in remote_updater.clients
assert remote_updater.next_worker()[0] == remote_updater.workers[1]
assert remote_updater.workers[1] in remote_updater.clients
assert len(remote_updater.clients) == 2
def test_next_worker_cycle(remote_updater: RemoteUpdater) -> None:
"""
must return first used worker if no free workers left
"""
worker1, client1 = remote_updater.next_worker()
worker2, client2 = remote_updater.next_worker()
assert remote_updater.next_worker() == (worker1, client1)
assert remote_updater.next_worker() == (worker2, client2)
assert remote_updater.next_worker() == (worker1, client1)
def test_partition(remote_updater: RemoteUpdater, mocker: MockerFixture) -> None:
"""
must partition as tree partition
"""
resolve_mock = mocker.patch("ahriman.core.tree.Tree.partition")
remote_updater.partition([])
resolve_mock.assert_called_once_with([], count=2)
def test_update(remote_updater: RemoteUpdater, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must process remote package updates
"""
worker, client = remote_updater.next_worker()
worker_mock = mocker.patch("ahriman.application.application.workers.remote_updater.RemoteUpdater.next_worker",
return_value=(worker, client))
request_mock = mocker.patch("ahriman.core.http.SyncAhrimanClient.make_request")
assert remote_updater.update([package_ahriman], Packagers("username"), bump_pkgrel=True) == Result()
worker_mock.assert_called_once_with()
request_mock.assert_called_once_with("POST", remote_updater._update_url(worker),
params=remote_updater.repository_id.query(),
json={
"increment": True,
"packager": "username",
"packages": [package_ahriman.base],
"patches": [],
"refresh": True,
}
)

View File

@ -0,0 +1,50 @@
import pytest
from ahriman.application.application.workers import Updater
from ahriman.application.application.workers.local_updater import LocalUpdater
from ahriman.application.application.workers.remote_updater import RemoteUpdater
from ahriman.core.configuration import Configuration
from ahriman.core.repository import Repository
from ahriman.models.worker import Worker
def test_load(configuration: Configuration, repository: Repository) -> None:
"""
must load local updater if empty worker list is set
"""
_, repository_id = configuration.check_loaded()
assert isinstance(Updater.load(repository_id, configuration, repository), LocalUpdater)
assert isinstance(Updater.load(repository_id, configuration, repository, []), LocalUpdater)
def test_load_from_option(configuration: Configuration, repository: Repository) -> None:
"""
must load remote updater if nonempty worker list is set
"""
_, repository_id = configuration.check_loaded()
assert isinstance(Updater.load(repository_id, configuration, repository, [Worker("remote")]), RemoteUpdater)
def test_load_from_configuration(configuration: Configuration, repository: Repository) -> None:
"""
must load remote updater from settings
"""
configuration.set_option("build", "workers", "remote")
_, repository_id = configuration.check_loaded()
assert isinstance(Updater.load(repository_id, configuration, repository), RemoteUpdater)
def test_partition(updater: Updater) -> None:
"""
must raise not implemented error for missing partition method
"""
with pytest.raises(NotImplementedError):
updater.partition([])
def test_update(updater: Updater) -> None:
"""
must raise not implemented error for missing update method
"""
with pytest.raises(NotImplementedError):
updater.update([])

View File

@ -101,8 +101,10 @@ def test_packages_add(spawner: Spawn, repository_id: RepositoryId, mocker: Mocke
must call package addition
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=False)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None)
assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
patches=[], now=False, increment=False, refresh=False)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
**{"username": None, "variable": [], "no-increment": ""})
def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@ -110,8 +112,10 @@ def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mo
must call package addition with update
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=True)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None, now="")
assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
patches=[], now=True, increment=False, refresh=False)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
**{"username": None, "variable": [], "no-increment": "", "now": ""})
def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@ -119,8 +123,10 @@ def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId,
must call package addition with username
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username", patches=[], now=False)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username="username")
assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username",
patches=[], now=False, increment=False, refresh=False)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
**{"username": "username", "variable": [], "no-increment": ""})
def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@ -129,9 +135,36 @@ def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId,
"""
patches = [PkgbuildPatch("key", "value"), PkgbuildPatch("key", "value")]
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=patches, now=False)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None,
variable=[patch.serialize() for patch in patches])
assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
patches=patches, now=False, increment=False, refresh=False)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
**{
"username": None,
"variable": [patch.serialize() for patch in patches],
"no-increment": ""
})
def test_packages_add_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
"""
must call package addition with increment
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
patches=[], now=False, increment=True, refresh=False)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
**{"username": None, "variable": [], "increment": ""})
def test_packages_add_with_refresh(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
"""
must call package addition with refresh
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
patches=[], now=False, increment=False, refresh=True)
spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
**{"username": None, "variable": [], "no-increment": "", "refresh": ""})
def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@ -139,9 +172,19 @@ def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: M
must call package rebuild
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_rebuild(repository_id, "python", "packager")
assert spawner.packages_rebuild(repository_id, "python", "packager", increment=False)
spawn_mock.assert_called_once_with(repository_id, "repo-rebuild",
**{"depends-on": "python", "username": "packager"})
**{"depends-on": "python", "username": "packager", "no-increment": ""})
def test_packages_rebuild_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
"""
must call package rebuild with increment
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_rebuild(repository_id, "python", "packager", increment=True)
spawn_mock.assert_called_once_with(repository_id, "repo-rebuild",
**{"depends-on": "python", "username": "packager", "increment": ""})
def test_packages_remove(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@ -159,27 +202,61 @@ def test_packages_update(spawner: Spawn, repository_id: RepositoryId, mocker: Mo
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_update(repository_id, "packager", aur=True, local=True, manual=True)
args = {"username": "packager", "aur": "", "local": "", "manual": ""}
assert spawner.packages_update(repository_id, "packager",
aur=True, local=True, manual=True, increment=False, refresh=False)
args = {"username": "packager", "aur": "", "local": "", "manual": "", "no-increment": ""}
spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
spawn_mock.reset_mock()
assert spawner.packages_update(repository_id, "packager", aur=False, local=True, manual=True)
args = {"username": "packager", "no-aur": "", "local": "", "manual": ""}
assert spawner.packages_update(repository_id, "packager",
aur=False, local=True, manual=True, increment=False, refresh=False)
args = {"username": "packager", "no-aur": "", "local": "", "manual": "", "no-increment": ""}
spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
spawn_mock.reset_mock()
assert spawner.packages_update(repository_id, "packager", aur=True, local=False, manual=True)
args = {"username": "packager", "aur": "", "no-local": "", "manual": ""}
assert spawner.packages_update(repository_id, "packager",
aur=True, local=False, manual=True, increment=False, refresh=False)
args = {"username": "packager", "aur": "", "no-local": "", "manual": "", "no-increment": ""}
spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
spawn_mock.reset_mock()
assert spawner.packages_update(repository_id, "packager", aur=True, local=True, manual=False)
args = {"username": "packager", "aur": "", "local": "", "no-manual": ""}
assert spawner.packages_update(repository_id, "packager",
aur=True, local=True, manual=False, increment=False, refresh=False)
args = {"username": "packager", "aur": "", "local": "", "no-manual": "", "no-increment": ""}
spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
spawn_mock.reset_mock()
def test_packages_update_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
"""
must call repo update with increment
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_update(repository_id, None,
aur=True, local=True, manual=True, increment=True, refresh=False)
spawn_mock.assert_called_once_with(repository_id, "repo-update",
**{"username": None, "aur": "", "local": "", "manual": "", "increment": ""})
def test_packages_update_with_refresh(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
"""
must call repo update with refresh
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
assert spawner.packages_update(repository_id, None,
aur=True, local=True, manual=True, increment=False, refresh=True)
spawn_mock.assert_called_once_with(repository_id, "repo-update",
**{
"username": None,
"aur": "",
"local": "",
"manual": "",
"no-increment": "",
"refresh": "",
}
)
def test_run(spawner: Spawn, mocker: MockerFixture) -> None:
"""
must implement run method

View File

@ -88,6 +88,13 @@ def test_tree_balance() -> None:
assert third == [leaf5]
def test_tree_balance_empty() -> None:
"""
must do not fail on empty tree balancing
"""
assert Tree.balance([]) == []
def test_tree_partition(package_ahriman: Package, package_python_schedule: Package) -> None:
"""
must partition dependencies tree

View File

@ -0,0 +1,10 @@
from ahriman.models.worker import Worker
def test_post_init() -> None:
"""
must read identifier from location if not set
"""
assert Worker("http://localhost:8080").identifier == "localhost:8080"
assert Worker("remote").identifier == "" # not a valid url
assert Worker("remote", identifier="id").identifier == "id"

View File

@ -0,0 +1 @@
# schema testing goes in view class tests

View File

@ -193,6 +193,9 @@ async def test_username(base: BaseView, mocker: MockerFixture) -> None:
policy = AsyncMock()
policy.identify.return_value = "identity"
mocker.patch("aiohttp.web.Application.get", return_value=policy)
json = AsyncMock()
json.return_value = {}
base._request = pytest.helpers.request(base.request.app, "", "", json=json)
assert await base.username() == "identity"
policy.identify.assert_called_once_with(base.request)
@ -202,4 +205,26 @@ async def test_username_no_auth(base: BaseView) -> None:
"""
must return None in case if auth is disabled
"""
json = AsyncMock()
json.return_value = {}
base._request = pytest.helpers.request(base.request.app, "", "", json=json)
assert await base.username() is None
async def test_username_request(base: BaseView) -> None:
"""
must read packager from request
"""
json = AsyncMock()
json.return_value = {"packager": "identity"}
base._request = pytest.helpers.request(base.request.app, "", "", json=json)
assert await base.username() == "identity"
async def test_username_request_exception(base: BaseView) -> None:
"""
must not fail in case if cannot read request
"""
assert await base.username() is None

View File

@ -41,7 +41,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
assert not request_schema.validate(payload)
response = await client.post("/api/v1/service/add", json=payload)
assert response.ok
add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=True)
add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
patches=[], now=True, increment=True, refresh=False)
json = await response.json()
assert json["process_id"] == "abc"
@ -74,7 +75,8 @@ async def test_post_patches(client: TestClient, repository_id: RepositoryId, moc
response = await client.post("/api/v1/service/add", json=payload)
assert response.ok
add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=True)
patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")],
now=True, increment=True, refresh=False)
async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:

View File

@ -40,7 +40,7 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
assert not request_schema.validate(payload)
response = await client.post("/api/v1/service/rebuild", json=payload)
assert response.ok
rebuild_mock.assert_called_once_with(repository_id, "python", "username")
rebuild_mock.assert_called_once_with(repository_id, "python", "username", increment=True)
json = await response.json()
assert json["process_id"] == "abc"

View File

@ -41,7 +41,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
assert not request_schema.validate(payload)
response = await client.post("/api/v1/service/request", json=payload)
assert response.ok
add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=False)
add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
patches=[], now=False, increment=False, refresh=False)
json = await response.json()
assert json["process_id"] == "abc"
@ -74,7 +75,8 @@ async def test_post_patches(client: TestClient, repository_id: RepositoryId, moc
response = await client.post("/api/v1/service/request", json=payload)
assert response.ok
add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=False)
patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")],
now=False, increment=False, refresh=False)
async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:

View File

@ -40,6 +40,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
"aur": True,
"local": True,
"manual": True,
"increment": True,
"refresh": False,
}
for payload in (