add workers autodicsovery feature

This commit is contained in:
2023-12-30 16:21:13 +02:00
parent fdf7a36271
commit c58fd3a4b9
40 changed files with 1489 additions and 16 deletions

View File

@ -39,6 +39,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository:
setup_mock = mocker.patch("ahriman.web.web.setup_server")
run_mock = mocker.patch("ahriman.web.web.run_server")
start_mock = mocker.patch("ahriman.core.spawn.Spawn.start")
trigger_mock = mocker.patch("ahriman.core.triggers.TriggerLoader.load")
stop_mock = mocker.patch("ahriman.core.spawn.Spawn.stop")
join_mock = mocker.patch("ahriman.core.spawn.Spawn.join")
_, repository_id = configuration.check_loaded()
@ -48,6 +49,8 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository:
setup_mock.assert_called_once_with(configuration, pytest.helpers.anyvar(int), [repository_id])
run_mock.assert_called_once_with(pytest.helpers.anyvar(int))
start_mock.assert_called_once_with()
trigger_mock.assert_called_once_with(repository_id, configuration)
trigger_mock().on_start.assert_called_once_with()
stop_mock.assert_called_once_with()
join_mock.assert_called_once_with()

View File

@ -1320,6 +1320,80 @@ def test_subparsers_service_tree_migrate(parser: argparse.ArgumentParser) -> Non
assert not args.report
def test_subparsers_service_worker_register(parser: argparse.ArgumentParser) -> None:
"""
service-worker-register command must imply trigger
"""
args = parser.parse_args(["service-worker-register"])
assert args.trigger == ["ahriman.core.distributed.WorkerRegisterTrigger"]
def test_subparsers_service_worker_register_option_architecture(parser: argparse.ArgumentParser) -> None:
"""
service-worker-register command must correctly parse architecture list
"""
args = parser.parse_args(["service-worker-register"])
assert args.architecture is None
args = parser.parse_args(["-a", "x86_64", "service-worker-register"])
assert args.architecture == "x86_64"
def test_subparsers_service_worker_register_option_repository(parser: argparse.ArgumentParser) -> None:
"""
service-worker-register command must correctly parse repository list
"""
args = parser.parse_args(["service-worker-register"])
assert args.repository is None
args = parser.parse_args(["-r", "repo", "service-worker-register"])
assert args.repository == "repo"
def test_subparsers_service_worker_register_repo_triggers(parser: argparse.ArgumentParser) -> None:
"""
service-worker-register must have same keys as repo-triggers
"""
args = parser.parse_args(["service-worker-register"])
reference_args = parser.parse_args(["repo-triggers"])
assert dir(args) == dir(reference_args)
def test_subparsers_service_worker_unregister(parser: argparse.ArgumentParser) -> None:
"""
service-worker-unregister command must imply trigger
"""
args = parser.parse_args(["service-worker-unregister"])
assert args.trigger == ["ahriman.core.distributed.WorkerUnregisterTrigger"]
def test_subparsers_service_worker_unregister_option_architecture(parser: argparse.ArgumentParser) -> None:
"""
service-worker-unregister command must correctly parse architecture list
"""
args = parser.parse_args(["service-worker-unregister"])
assert args.architecture is None
args = parser.parse_args(["-a", "x86_64", "service-worker-unregister"])
assert args.architecture == "x86_64"
def test_subparsers_service_worker_unregister_option_repository(parser: argparse.ArgumentParser) -> None:
"""
service-worker-unregister command must correctly parse repository list
"""
args = parser.parse_args(["service-worker-unregister"])
assert args.repository is None
args = parser.parse_args(["-r", "repo", "service-worker-unregister"])
assert args.repository == "repo"
def test_subparsers_service_worker_unregister_repo_triggers(parser: argparse.ArgumentParser) -> None:
"""
service-worker-unregister must have same keys as repo-triggers
"""
args = parser.parse_args(["service-worker-unregister"])
reference_args = parser.parse_args(["repo-triggers"])
assert dir(args) == dir(reference_args)
def test_subparsers_user_add(parser: argparse.ArgumentParser) -> None:
"""
user-add command must imply action, architecture, exit code, lock, quiet, report and repository

View File

@ -0,0 +1,8 @@
from ahriman.core.database.migrations.m013_workers import steps
def test_migration_workers() -> None:
"""
migration must not be empty
"""
assert steps

View File

@ -0,0 +1,46 @@
from ahriman.core.database import SQLite
from ahriman.models.worker import Worker
def test_workers_get_insert(database: SQLite) -> None:
"""
must insert workers to database
"""
database.workers_insert(Worker("address1", identifier="1"))
database.workers_insert(Worker("address2", identifier="2"))
assert database.workers_get() == [
Worker("address1", identifier="1"), Worker("address2", identifier="2")
]
def test_workers_insert_remove(database: SQLite) -> None:
"""
must remove worker from database
"""
database.workers_insert(Worker("address1", identifier="1"))
database.workers_insert(Worker("address2", identifier="2"))
database.workers_remove("1")
assert database.workers_get() == [Worker("address2", identifier="2")]
def test_workers_insert_remove_all(database: SQLite) -> None:
"""
must remove all workers
"""
database.workers_insert(Worker("address1", identifier="1"))
database.workers_insert(Worker("address2", identifier="2"))
database.workers_remove()
assert database.workers_get() == []
def test_workers_insert_insert(database: SQLite) -> None:
"""
must update worker in database
"""
database.workers_insert(Worker("address1", identifier="1"))
assert database.workers_get() == [Worker("address1", identifier="1")]
database.workers_insert(Worker("address2", identifier="1"))
assert database.workers_get() == [Worker("address2", identifier="1")]

View File

@ -0,0 +1,20 @@
import pytest
from ahriman.core.configuration import Configuration
from ahriman.core.distributed.distributed_system import DistributedSystem
@pytest.fixture
def distributed_system(configuration: Configuration) -> DistributedSystem:
"""
distributed system fixture
Args:
configuration(Configuration): configuration fixture
Returns:
DistributedSystem: distributed system test instance
"""
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
return DistributedSystem(repository_id, configuration)

View File

@ -0,0 +1,176 @@
import json
import requests
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.distributed.distributed_system import DistributedSystem
from ahriman.models.worker import Worker
def test_identifier_path(configuration: Configuration) -> None:
"""
must correctly set default identifier path
"""
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
assert DistributedSystem(repository_id, configuration).identifier_path
def test_configuration_sections(configuration: Configuration) -> None:
"""
must correctly parse target list
"""
assert DistributedSystem.configuration_sections(configuration) == ["worker"]
def test_workers_url(distributed_system: DistributedSystem) -> None:
"""
must generate workers url correctly
"""
assert distributed_system._workers_url().startswith(distributed_system.address)
assert distributed_system._workers_url().endswith("/api/v1/distributed")
assert distributed_system._workers_url("id").startswith(distributed_system.address)
assert distributed_system._workers_url("id").endswith("/api/v1/distributed/id")
def test_load_identifier(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must generate identifier
"""
mocker.patch("pathlib.Path.is_file", return_value=False)
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
system = DistributedSystem(repository_id, configuration)
assert system.load_identifier(configuration, "worker")
def test_load_identifier_configuration(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must load identifier from configuration
"""
identifier = "id"
mocker.patch("pathlib.Path.is_file", return_value=False)
configuration.set_option("worker", "identifier", identifier)
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
system = DistributedSystem(repository_id, configuration)
assert system.worker.identifier == identifier
def test_load_identifier_filesystem(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must load identifier from filesystem
"""
identifier = "id"
mocker.patch("pathlib.Path.is_file", return_value=True)
read_mock = mocker.patch("pathlib.Path.read_text", return_value=identifier)
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
system = DistributedSystem(repository_id, configuration)
assert system.worker.identifier == identifier
read_mock.assert_called_once_with(encoding="utf8")
def test_register(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must register service
"""
mocker.patch("pathlib.Path.is_file", return_value=False)
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
write_mock = mocker.patch("pathlib.Path.write_text")
distributed_system.register()
run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed",
json=distributed_system.worker.view())
write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8")
assert distributed_system._owe_identifier
def test_register_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must skip service registration if it doesn't owe the identifier
"""
mocker.patch("pathlib.Path.is_file", return_value=True)
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
write_mock = mocker.patch("pathlib.Path.write_text")
distributed_system.register()
run_mock.assert_not_called()
write_mock.assert_not_called()
assert not distributed_system._owe_identifier
def test_register_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must register service even if it doesn't owe the identifier if force is supplied
"""
mocker.patch("pathlib.Path.is_file", return_value=True)
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
write_mock = mocker.patch("pathlib.Path.write_text")
distributed_system.register(force=True)
run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed",
json=distributed_system.worker.view())
write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8")
assert distributed_system._owe_identifier
def test_unregister(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must unregister service
"""
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
remove_mock = mocker.patch("pathlib.Path.unlink")
distributed_system._owe_identifier = True
distributed_system.unregister()
run_mock.assert_called_once_with(
"DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}")
remove_mock.assert_called_once_with(missing_ok=True)
def test_unregister_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must skip service removal if it doesn't owe the identifier
"""
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
remove_mock = mocker.patch("pathlib.Path.unlink")
distributed_system.unregister()
run_mock.assert_not_called()
remove_mock.assert_not_called()
def test_unregister_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must remove service even if it doesn't owe the identifier if force is supplied
"""
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
remove_mock = mocker.patch("pathlib.Path.unlink")
distributed_system.unregister(force=True)
run_mock.assert_called_once_with(
"DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}")
remove_mock.assert_called_once_with(missing_ok=True)
def test_workers_get(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must return available remote workers
"""
worker = Worker("remote")
response_obj = requests.Response()
response_obj._content = json.dumps([worker.view()]).encode("utf8")
response_obj.status_code = 200
requests_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request",
return_value=response_obj)
result = distributed_system.workers()
requests_mock.assert_called_once_with("GET", distributed_system._workers_url())
assert result == [worker]

View File

@ -0,0 +1,34 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.distributed import WorkerLoaderTrigger
from ahriman.models.worker import Worker
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must load workers from remote
"""
worker = Worker("address")
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerLoaderTrigger.workers", return_value=[worker])
_, repository_id = configuration.check_loaded()
trigger = WorkerLoaderTrigger(repository_id, configuration)
trigger.on_start()
run_mock.assert_called_once_with()
assert configuration.getlist("build", "workers") == [worker.address]
def test_on_start_skip(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must skip loading if option is already set
"""
configuration.set_option("status", "address", "http://localhost:8081")
configuration.set_option("build", "workers", "address")
run_mock = mocker.patch("ahriman.core.distributed.WorkerLoaderTrigger.workers")
_, repository_id = configuration.check_loaded()
trigger = WorkerLoaderTrigger(repository_id, configuration)
trigger.on_start()
run_mock.assert_not_called()

View File

@ -0,0 +1,17 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.distributed import WorkerRegisterTrigger
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must register itself as worker
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerRegisterTrigger.register")
_, repository_id = configuration.check_loaded()
trigger = WorkerRegisterTrigger(repository_id, configuration)
trigger.on_start()
run_mock.assert_called_once_with(force=True)

View File

@ -0,0 +1,30 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.distributed import WorkerTrigger
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must register itself as worker
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.register")
_, repository_id = configuration.check_loaded()
trigger = WorkerTrigger(repository_id, configuration)
trigger.on_start()
run_mock.assert_called_once_with()
def test_on_stop(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must unregister itself as worker
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.unregister")
_, repository_id = configuration.check_loaded()
trigger = WorkerTrigger(repository_id, configuration)
trigger.on_stop()
run_mock.assert_called_once_with()

View File

@ -0,0 +1,17 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.distributed import WorkerUnregisterTrigger
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must unregister itself as worker
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerUnregisterTrigger.unregister")
_, repository_id = configuration.check_loaded()
trigger = WorkerUnregisterTrigger(repository_id, configuration)
trigger.on_start()
run_mock.assert_called_once_with(force=True)

View File

@ -10,6 +10,7 @@ from ahriman.models.changes import Changes
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.worker import Worker
def test_load(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
@ -227,3 +228,40 @@ def test_status_update(watcher: Watcher) -> None:
"""
watcher.status_update(BuildStatusEnum.Success)
assert watcher.status.status == BuildStatusEnum.Success
def test_workers_get(watcher: Watcher, mocker: MockerFixture) -> None:
"""
must retrieve workers
"""
worker = Worker("remote")
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_get", return_value=[worker])
assert watcher.workers_get() == [worker]
worker_mock.assert_called_once_with()
def test_workers_remove(watcher: Watcher, mocker: MockerFixture) -> None:
"""
must remove workers
"""
identifier = "identifier"
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_remove")
watcher.workers_remove(identifier)
watcher.workers_remove()
worker_mock.assert_has_calls([
MockCall(identifier),
MockCall(None),
])
def test_workers_update(watcher: Watcher, mocker: MockerFixture) -> None:
"""
must update workers
"""
worker = Worker("remote")
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_insert")
watcher.workers_update(worker)
worker_mock.assert_called_once_with(worker)

View File

@ -12,6 +12,7 @@ from ahriman.models.changes import Changes
from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.worker import Worker
def test_parse_address(configuration: Configuration) -> None:
@ -32,14 +33,6 @@ def test_parse_address(configuration: Configuration) -> None:
assert WebClient.parse_address(configuration) == ("status", "http://localhost:8082")
def test_status_url(web_client: WebClient) -> None:
"""
must generate package status url correctly
"""
assert web_client._status_url().startswith(web_client.address)
assert web_client._status_url().endswith("/api/v1/status")
def test_changes_url(web_client: WebClient, package_ahriman: Package) -> None:
"""
must generate changes url correctly
@ -67,6 +60,14 @@ def test_package_url(web_client: WebClient, package_ahriman: Package) -> None:
assert web_client._package_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}")
def test_status_url(web_client: WebClient) -> None:
"""
must generate package status url correctly
"""
assert web_client._status_url().startswith(web_client.address)
assert web_client._status_url().endswith("/api/v1/status")
def test_package_add(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must process package addition

View File

@ -8,3 +8,17 @@ def test_post_init() -> None:
assert Worker("http://localhost:8080").identifier == "localhost:8080"
assert Worker("remote").identifier == "" # not a valid url
assert Worker("remote", identifier="id").identifier == "id"
def test_view() -> None:
"""
must generate json view
"""
worker = Worker("address")
assert worker.view() == {"address": worker.address, "identifier": worker.identifier}
worker = Worker("http://localhost:8080")
assert worker.view() == {"address": worker.address, "identifier": worker.identifier}
worker = Worker("http://localhost:8080", identifier="abc")
assert worker.view() == {"address": worker.address, "identifier": worker.identifier}

View File

@ -0,0 +1 @@
# schema testing goes in view class tests

View File

@ -0,0 +1 @@
# schema testing goes in view class tests

View File

@ -0,0 +1,70 @@
import pytest
from aiohttp.test_utils import TestClient
from ahriman.models.user_access import UserAccess
from ahriman.models.worker import Worker
from ahriman.web.views.v1.distributed.worker import WorkerView
async def test_get_permission() -> None:
"""
must return correct permission for the request
"""
for method in ("DELETE", "GET"):
request = pytest.helpers.request("", "", method)
assert await WorkerView.get_permission(request) == UserAccess.Full
def test_routes() -> None:
"""
must return correct routes
"""
assert WorkerView.ROUTES == ["/api/v1/distributed/{identifier}"]
async def test_delete(client: TestClient) -> None:
"""
must delete single worker
"""
await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"})
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
response = await client.delete("/api/v1/distributed/1")
assert response.status == 204
response = await client.get("/api/v1/distributed/1")
assert response.status == 404
response = await client.get("/api/v1/distributed/2")
assert response.ok
async def test_get(client: TestClient) -> None:
"""
must return specific worker
"""
worker = Worker("address1", identifier="1")
await client.post("/api/v1/distributed", json=worker.view())
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
response_schema = pytest.helpers.schema_response(WorkerView.get)
response = await client.get(f"/api/v1/distributed/{worker.identifier}")
assert response.ok
json = await response.json()
assert not response_schema.validate(json, many=True)
workers = [Worker(item["address"], identifier=item["identifier"]) for item in json]
assert workers == [worker]
async def test_get_not_found(client: TestClient) -> None:
"""
must return Not Found for unknown package
"""
response_schema = pytest.helpers.schema_response(WorkerView.get, code=404)
response = await client.get("/api/v1/distributed/1")
assert response.status == 404
assert not response_schema.validate(await response.json())

View File

@ -0,0 +1,83 @@
import pytest
from aiohttp.test_utils import TestClient
from ahriman.models.user_access import UserAccess
from ahriman.models.worker import Worker
from ahriman.web.views.v1.distributed.workers import WorkersView
async def test_get_permission() -> None:
"""
must return correct permission for the request
"""
for method in ("DELETE", "GET", "POST"):
request = pytest.helpers.request("", "", method)
assert await WorkersView.get_permission(request) == UserAccess.Full
def test_routes() -> None:
"""
must return correct routes
"""
assert WorkersView.ROUTES == ["/api/v1/distributed"]
async def test_delete(client: TestClient) -> None:
"""
must delete all workers
"""
await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"})
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
response = await client.delete("/api/v1/distributed")
assert response.status == 204
response = await client.get("/api/v1/distributed")
json = await response.json()
assert not json
async def test_get(client: TestClient) -> None:
"""
must return all workers
"""
await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"})
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
response_schema = pytest.helpers.schema_response(WorkersView.get)
response = await client.get("/api/v1/distributed")
assert response.ok
json = await response.json()
assert not response_schema.validate(json, many=True)
workers = [Worker(item["address"], identifier=item["identifier"]) for item in json]
assert workers == [Worker("address1", identifier="1"), Worker("address2", identifier="2")]
async def test_post(client: TestClient) -> None:
"""
must update worker
"""
worker = Worker("address1", identifier="1")
request_schema = pytest.helpers.schema_request(WorkersView.post)
payload = worker.view()
assert not request_schema.validate(payload)
response = await client.post("/api/v1/distributed", json=payload)
assert response.status == 204
response = await client.get(f"/api/v1/distributed/{worker.identifier}")
assert response.ok
async def test_post_exception(client: TestClient) -> None:
"""
must raise exception on invalid payload
"""
response_schema = pytest.helpers.schema_response(WorkersView.post, code=400)
response = await client.post("/api/v1/distributed", json={})
assert response.status == 400
assert not response_schema.validate(await response.json())

View File

@ -115,4 +115,7 @@ username = arcan1s
enable_archive_upload = yes
host = 127.0.0.1
static_path = ../web/templates/static
templates = ../web/templates
templates = ../web/templates
[worker]
address = http://localhost:8081